You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wy...@apache.org on 2023/06/02 20:46:30 UTC
[asterixdb] branch master updated: [ASTERIXDB-3202][STO] Clean dangling LAF
This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bf9e7dfc59 [ASTERIXDB-3202][STO] Clean dangling LAF
bf9e7dfc59 is described below
commit bf9e7dfc59f601aa9ec510587b6cb5eb44679916
Author: Wail Alkowaileet <wa...@gmail.com>
AuthorDate: Wed May 31 09:15:06 2023 -0700
[ASTERIXDB-3202][STO] Clean dangling LAF
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Clean LAF when its corresponding on-disk
component is missing.
Change-Id: I19d2177f8e77c47ae256855a0d599fa19a2625bd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17584
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wa...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../am/lsm/btree/impls/LSMBTreeFileManager.java | 14 +++++----
.../common/impls/AbstractLSMIndexFileManager.java | 21 +++++++++++--
.../storage/am/lsm/btree/LSMBTreeBulkLoadTest.java | 3 +-
.../lsm/btree/LSMBTreeComponentLifecycleTest.java | 5 ++--
.../storage/am/lsm/btree/LSMBTreeDeleteTest.java | 3 +-
.../am/lsm/btree/LSMBTreeFileManagerTest.java | 21 +++++++++----
.../am/lsm/btree/LSMBTreeFilterMergeTest.java | 3 +-
.../storage/am/lsm/btree/LSMBTreeInsertTest.java | 3 +-
.../am/lsm/btree/LSMBTreeLifecycleTest.java | 8 ++---
.../am/lsm/btree/LSMBTreeMergeFailTest.java | 3 +-
.../storage/am/lsm/btree/LSMBTreeMergeTest.java | 3 +-
.../am/lsm/btree/LSMBTreeMultiBulkLoadTest.java | 3 +-
.../lsm/btree/LSMBTreePageWriteCallbackTest.java | 9 ++++--
.../lsm/btree/LSMBTreeScanDiskComponentsTest.java | 3 +-
...SMBTreeUpdateInPlaceScanDiskComponentsTest.java | 34 +++++++++++-----------
.../storage/am/lsm/btree/LSMBTreeUpdateTest.java | 3 +-
.../am/lsm/btree/util/LSMBTreeTestContext.java | 9 +++---
.../am/lsm/btree/util/LSMBTreeTestHarness.java | 12 ++++++++
.../common/component/TestLsmIndexFileManager.java | 4 +--
19 files changed, 108 insertions(+), 56 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 08eba0df68..98b82db8be 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -77,16 +78,19 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
List<LSMComponentFileReferences> validFiles = new ArrayList<>();
- ArrayList<IndexComponentFileReference> allBTreeFiles = new ArrayList<>();
- ArrayList<IndexComponentFileReference> allBloomFilterFiles = new ArrayList<>();
+ List<IndexComponentFileReference> allBTreeFiles = new ArrayList<>();
+ List<IndexComponentFileReference> allBloomFilterFiles = new ArrayList<>();
// List of valid BTree files.
cleanupAndGetValidFilesInternal(BTREE_FILTER, btreeFactory, allBTreeFiles, btreeFactory.getBufferCache());
- HashSet<String> btreeFilesSet = new HashSet<>();
+ Set<String> btreeFilesSet = new HashSet<>();
for (IndexComponentFileReference cmpFileName : allBTreeFiles) {
int index = cmpFileName.getFileName().lastIndexOf(DELIMITER);
btreeFilesSet.add(cmpFileName.getFileName().substring(0, index));
}
+ // Clean up LAFs if any
+ cleanLookAsideFiles(btreeFilesSet, btreeFactory.getBufferCache());
+
if (hasBloomFilter) {
validateFiles(btreeFilesSet, allBloomFilterFiles, BLOOM_FILTER_FILTER, null, btreeFactory.getBufferCache());
// Sanity check.
@@ -156,11 +160,11 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
// Sort valid files in reverse lexicographical order, such that newer
// files come first.
- Collections.sort(validComparableBTreeFiles, recencyCmp);
+ validComparableBTreeFiles.sort(recencyCmp);
Iterator<IndexComponentFileReference> btreeFileIter = validComparableBTreeFiles.iterator();
Iterator<IndexComponentFileReference> bloomFilterFileIter = null;
if (hasBloomFilter) {
- Collections.sort(validComparableBloomFilterFiles, recencyCmp);
+ validComparableBloomFilterFiles.sort(recencyCmp);
bloomFilterFileIter = validComparableBloomFilterFiles.iterator();
}
IndexComponentFileReference cmpBTreeFileName = null;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 610232f812..6194854bbc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -23,7 +23,6 @@ import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -82,6 +81,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage
public static final FilenameFilter COMPONENT_FILES_FILTER = (dir, name) -> !name.startsWith(".");
protected static FilenameFilter BLOOM_FILTER_FILTER =
(dir, name) -> !name.startsWith(".") && name.endsWith(BLOOM_FILTER_SUFFIX);
+ protected static FilenameFilter LAF_FILTER = (dir, name) -> !name.startsWith(".") && name.endsWith(LAF_SUFFIX);
protected static final Comparator<String> cmp = new FileNameComparator();
protected final IIOManager ioManager;
@@ -137,7 +137,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage
}
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
- TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<IndexComponentFileReference> allFiles,
+ TreeIndexFactory<? extends ITreeIndex> treeFactory, List<IndexComponentFileReference> allFiles,
IBufferCache bufferCache) throws HyracksDataException {
Set<FileReference> files = ioManager.list(baseDir, filter);
for (FileReference filePath : files) {
@@ -155,7 +155,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage
}
}
- protected void validateFiles(HashSet<String> groundTruth, ArrayList<IndexComponentFileReference> validFiles,
+ protected void validateFiles(Set<String> groundTruth, List<IndexComponentFileReference> validFiles,
FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, IBufferCache bufferCache)
throws HyracksDataException {
ArrayList<IndexComponentFileReference> tmpAllInvListsFiles = new ArrayList<>();
@@ -307,6 +307,21 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage
return baseDir.getChild(name);
}
+ protected void cleanLookAsideFiles(Set<String> groundTruth, IBufferCache bufferCache) throws HyracksDataException {
+ ICompressorDecompressor compDecomp = compressorDecompressorFactory.createInstance();
+ if (compDecomp == NoOpCompressorDecompressor.INSTANCE) {
+ return;
+ }
+
+ List<IndexComponentFileReference> allLookAsideFiles = new ArrayList<>();
+ cleanupAndGetValidFilesInternal(LAF_FILTER, null, allLookAsideFiles, null);
+ for (IndexComponentFileReference laf : allLookAsideFiles) {
+ if (!groundTruth.contains(laf.getSequence())) {
+ delete(bufferCache, laf.getFileRef());
+ }
+ }
+ }
+
private boolean isCompressible(String fileName) {
return !fileName.endsWith(BLOOM_FILTER_SUFFIX) && !fileName.endsWith(DELETE_TREE_SUFFIX);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 28b0139130..1477535035 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -57,7 +57,8 @@ public class LSMBTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
index 7f8fd8a7f9..8608caccee 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeComponentLifecycleTest.java
@@ -82,7 +82,8 @@ public class LSMBTreeComponentLifecycleTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
scheduler, ioCallbackFactory, harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
private OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys)
@@ -425,7 +426,7 @@ public class LSMBTreeComponentLifecycleTest {
private class User extends SingleThreadEventProcessor<Request> {
- private Semaphore step = new Semaphore(0);
+ private final Semaphore step = new Semaphore(0);
public User(String username) {
super(username);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
index 857a20cbf1..fb25ee7cbe 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeDeleteTest.java
@@ -57,7 +57,8 @@ public class LSMBTreeDeleteTest extends OrderedIndexDeleteTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
index a6ba178e1e..c607caeffa 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFileManagerTest.java
@@ -22,6 +22,7 @@ import java.io.File;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
@@ -30,6 +31,7 @@ import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDisk
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.common.compression.file.CompressedFileReference;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -56,7 +58,8 @@ public class LSMBTreeFileManagerTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, 1,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
ctx.getIndex().create();
ctx.getIndex().activate();
@@ -80,16 +83,22 @@ public class LSMBTreeFileManagerTest {
ctx.getIndex().deactivate();
// Delete the btree file and keep the bloom filter file from the disk component
- LSMBTreeWithBloomFilterDiskComponent ilsmDiskComponent =
+ LSMBTreeWithBloomFilterDiskComponent diskComponent =
(LSMBTreeWithBloomFilterDiskComponent) btree.getDiskComponents().get(0);
- ilsmDiskComponent.getIndex().getFileReference().delete();
+ CompressedFileReference fileReference = (CompressedFileReference) diskComponent.getIndex().getFileReference();
- File bloomFilterFile = ilsmDiskComponent.getBloomFilter().getFileReference().getFile().getAbsoluteFile();
- Assert.assertEquals("Check bloom filter file exists", true, bloomFilterFile.exists());
+ // Only delete the index file
+ IoUtil.delete(fileReference.getFile());
+ Assert.assertTrue("Check LAF exists", fileReference.getLAFFileReference().getFile().exists());
+
+ File bloomFilterFile = diskComponent.getBloomFilter().getFileReference().getFile().getAbsoluteFile();
+ Assert.assertTrue("Check bloom filter file exists", bloomFilterFile.exists());
// Activating the index again should delete the orphaned bloom filter file as well as the disk component
ctx.getIndex().activate();
- Assert.assertEquals("Check bloom filter file deleted", false, bloomFilterFile.exists());
+ Assert.assertFalse("Check bloom filter file deleted", bloomFilterFile.exists());
Assert.assertEquals("Check disk components", 0, btree.getDiskComponents().size());
+ // After index activation, the dangling LAF should be removed
+ Assert.assertFalse("Check LAF deleted", fileReference.getLAFFileReference().getFile().exists());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
index 777830816a..bf76e2026e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeFilterMergeTest.java
@@ -56,7 +56,8 @@ public class LSMBTreeFilterMergeTest extends LSMBTreeFilterMergeTestDriver {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
index 69760bfe30..dace7316cc 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeInsertTest.java
@@ -57,7 +57,8 @@ public class LSMBTreeInsertTest extends OrderedIndexInsertTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
index 431abb8a18..d131ccb124 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeLifecycleTest.java
@@ -43,10 +43,7 @@ public class LSMBTreeLifecycleTest extends AbstractIndexLifecycleTest {
@Override
protected boolean persistentStateExists() throws Exception {
// make sure all of the directories exist
- if (!harness.getFileReference().getFile().exists()) {
- return false;
- }
- return true;
+ return harness.getFileReference().getFile().exists();
}
@Override
@@ -61,7 +58,8 @@ public class LSMBTreeLifecycleTest extends AbstractIndexLifecycleTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, fieldSerdes.length,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
index = testCtx.getIndex();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
index bd5d97d106..bfa9ba676b 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeFailTest.java
@@ -105,7 +105,8 @@ public class LSMBTreeMergeFailTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
scheduler, harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
- harness.getMetadataPageManagerFactory(), filtered, true, false);
+ harness.getMetadataPageManagerFactory(), filtered, true, false,
+ harness.getCompressorDecompressorFactory());
}
private class TestIoScheduler implements ILSMIOOperationScheduler {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
index c9a2a999de..b21a8b9faf 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMergeTest.java
@@ -56,7 +56,8 @@ public class LSMBTreeMergeTest extends LSMBTreeMergeTestDriver {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), filtered, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
index 07cd08ff99..d131b1fce0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -58,7 +58,8 @@ public class LSMBTreeMultiBulkLoadTest extends OrderedIndexBulkLoadTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
index 4e59ab0f5e..cdf1500500 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreePageWriteCallbackTest.java
@@ -44,6 +44,7 @@ import org.apache.hyracks.storage.common.IResource;
import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
import org.apache.hyracks.storage.common.buffercache.IRateLimiter;
import org.apache.hyracks.storage.common.buffercache.SleepRateLimiter;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -53,14 +54,15 @@ public class LSMBTreePageWriteCallbackTest extends OrderedIndexTestDriver {
private final OrderedIndexTestUtils orderedIndexTestUtils;
- private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+ // Uses NoOp compressor/decompressor to mitigate assertions against number of pages
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness(NoOpCompressorDecompressorFactory.INSTANCE);
private final int PAGES_PER_FORCE = 16;
private int pageCounter = 0;
private LSMIndexPageWriteCallback lastCallback = null;
private final IRateLimiter testLimiter = new IRateLimiter() {
- IRateLimiter limiter = SleepRateLimiter.create(100 * 1000);
+ final IRateLimiter limiter = SleepRateLimiter.create(100 * 1000);
@Override
public void setRate(double ratePerSecond) {
@@ -112,7 +114,8 @@ public class LSMBTreePageWriteCallbackTest extends OrderedIndexTestDriver {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(), pageWriteCallbackFactory,
- harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
index c9c87bdcfe..4f2b2f7ec3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeScanDiskComponentsTest.java
@@ -80,7 +80,8 @@ public class LSMBTreeScanDiskComponentsTest extends OrderedIndexTestDriver {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
index 8d824052ec..0125318f9c 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceScanDiskComponentsTest.java
@@ -1,20 +1,20 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.hyracks.storage.am.lsm.btree;
@@ -124,7 +124,7 @@ public class LSMBTreeUpdateInPlaceScanDiskComponentsTest extends OrderedIndexTes
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, !hasOnlyKeys,
- hasOnlyKeys);
+ hasOnlyKeys, harness.getCompressorDecompressorFactory());
}
interface IndexModification {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
index ceb8c11fc6..de2dfa54a0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateTest.java
@@ -57,7 +57,8 @@ public class LSMBTreeUpdateTest extends OrderedIndexUpdateTest {
harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, numKeys,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false);
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), false, true, false,
+ harness.getCompressorDecompressorFactory());
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 928f60ce94..82f6f4baf0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -22,6 +22,7 @@ package org.apache.hyracks.storage.am.lsm.btree.util;
import java.util.Collection;
import java.util.List;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -42,7 +43,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
import org.apache.hyracks.util.trace.ITraceCategoryRegistry;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.TraceCategoryRegistry;
@@ -82,7 +82,8 @@ public final class LSMBTreeTestContext extends OrderedIndexTestContext {
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered, boolean needKeyDupCheck,
- boolean updateAware) throws HyracksDataException {
+ boolean updateAware, ICompressorDecompressorFactory compressorDecompressorFactory)
+ throws HyracksDataException {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
int[] bloomFilterKeyFields = new int[numKeyFields];
@@ -103,7 +104,7 @@ public final class LSMBTreeTestContext extends OrderedIndexTestContext {
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, needKeyDupCheck, filterTypeTraits,
filterCmp, btreefields, filterfields, true, metadataPageManagerFactory, updateAware, ITracer.NONE,
- NoOpCompressorDecompressorFactory.INSTANCE, true, null, null);
+ compressorDecompressorFactory, true, null, null);
} else {
lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
@@ -111,7 +112,7 @@ public final class LSMBTreeTestContext extends OrderedIndexTestContext {
true, metadataPageManagerFactory,
updateAware, new Tracer(LSMBTreeTestContext.class.getSimpleName(),
ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()),
- NoOpCompressorDecompressorFactory.INSTANCE, true, null, null);
+ compressorDecompressorFactory, true, null, null);
}
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered);
return testCtx;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
index d64ea304f2..4d2b086848 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestHarness.java
@@ -27,6 +27,7 @@ import java.util.Date;
import java.util.List;
import java.util.Random;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -50,6 +51,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ThreadCountingTracker;
import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.compression.SnappyCompressorDecompressorFactory;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
import org.apache.logging.log4j.LogManager;
@@ -71,6 +73,7 @@ public class LSMBTreeTestHarness {
protected final int hyracksFrameSize;
protected final double bloomFilterFalsePositiveRate;
protected final int numMutableComponents;
+ private final ICompressorDecompressorFactory compressorDecompressorFactory;
protected IOManager ioManager;
protected int ioDeviceId;
@@ -91,6 +94,10 @@ public class LSMBTreeTestHarness {
protected FileReference file;
public LSMBTreeTestHarness() {
+ this(new SnappyCompressorDecompressorFactory());
+ }
+
+ public LSMBTreeTestHarness(ICompressorDecompressorFactory compressorDecompressorFactory) {
this.diskPageSize = AccessMethodTestsConfig.LSM_BTREE_DISK_PAGE_SIZE;
this.diskNumPages = AccessMethodTestsConfig.LSM_BTREE_DISK_NUM_PAGES;
this.diskMaxOpenFiles = AccessMethodTestsConfig.LSM_BTREE_DISK_MAX_OPEN_FILES;
@@ -105,6 +112,7 @@ public class LSMBTreeTestHarness {
this.metadataPageManagerFactory = AppendOnlyLinkedMetadataPageManagerFactory.INSTANCE;
this.ioOpCallbackFactory = new CountingIoOperationCallbackFactory();
this.pageWriteCallbackFactory = NoOpPageWriteCallbackFactory.INSTANCE;
+ this.compressorDecompressorFactory = compressorDecompressorFactory;
}
public void setUp() throws HyracksDataException {
@@ -224,4 +232,8 @@ public class LSMBTreeTestHarness {
public IMetadataPageManagerFactory getMetadataPageManagerFactory() {
return metadataPageManagerFactory;
}
+
+ public ICompressorDecompressorFactory getCompressorDecompressorFactory() {
+ return compressorDecompressorFactory;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
index c255ee528a..9c65703d14 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
@@ -20,7 +20,7 @@
package org.apache.hyracks.storage.am.lsm.common.component;
import java.io.FilenameFilter;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -42,7 +42,7 @@ public class TestLsmIndexFileManager extends AbstractLSMIndexFileManager {
@Override
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
- TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<IndexComponentFileReference> allFiles,
+ TreeIndexFactory<? extends ITreeIndex> treeFactory, List<IndexComponentFileReference> allFiles,
IBufferCache bufferCache) {
String[] files = baseDir.getFile().list(filter);
for (String fileName : files) {