You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2019/04/09 19:35:38 UTC
[asterixdb] 04/05: [NO ISSUE][STO] Ensure Invalid Merged Components
Are Deleted
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 153b2ce970423d9fa0f02f83951d9b854b4732e6
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Sun Apr 7 21:11:52 2019 +0300
[NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When cleaning up an index's invalid components, check the
components end sequence in addition to the start sequence
to ensure invalid merged components with are cleaned too.
- Add test case.
Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3331
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../PersistentLocalResourceRepositoryTest.java | 30 ++++++++++++----------
.../PersistentLocalResourceRepository.java | 8 +++---
2 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
index 54ae683..00d2d3d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.storage;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -127,19 +128,9 @@ public class PersistentLocalResourceRepositoryTest {
String invalidComponentRange = invalidComponentId + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath);
String indexDir = indexDirRef.getFile().getAbsolutePath();
- // create the invalid component files
- Path btreePath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER
- + AbstractLSMIndexFileManager.BTREE_SUFFIX);
- Path filterPath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER
- + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
- Files.createFile(btreePath);
- Files.createFile(filterPath);
-
- // clean up the index partition
- localResourceRepository.cleanup(lr.getPartition());
- // ensure that the invalid component was deleted
- Assert.assertFalse(btreePath.toFile().exists());
- Assert.assertFalse(filterPath.toFile().exists());
+ ensureInvalidComponentDeleted(indexDir, invalidComponentRange, localResourceRepository, lr);
+ String invalidMergeComponentRange = "0" + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
+ ensureInvalidComponentDeleted(indexDir, invalidMergeComponentRange, localResourceRepository, lr);
// ensure that valid components still exist
// find index valid component timestamp from checkpoint
@@ -179,4 +170,17 @@ public class PersistentLocalResourceRepositoryTest {
Assert.assertFalse(indexMetadataFile.exists());
Assert.assertFalse(indexMetadataMaskFile.exists());
}
+
+ private void ensureInvalidComponentDeleted(String indexDir, String componentSeq,
+ PersistentLocalResourceRepository localResourceRepository, DatasetLocalResource lr) throws IOException {
+ Path btreePath = Paths.get(indexDir,
+ componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BTREE_SUFFIX);
+ Path filterPath = Paths.get(indexDir,
+ componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
+ Files.createFile(btreePath);
+ Files.createFile(filterPath);
+ localResourceRepository.cleanup(lr.getPartition());
+ Assert.assertFalse(btreePath.toFile().exists());
+ Assert.assertFalse(filterPath.toFile().exists());
+ }
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 8f870c0..aef7bbd 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -497,10 +497,12 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
}
final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence();
for (File componentFile : indexComponentFiles) {
- // delete any file with start sequence > valid component sequence
+ // delete any file with start or end sequence > valid component sequence
final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart();
- if (fileStart > validComponentSequence) {
- LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath());
+ final long fileEnd = IndexComponentFileReference.of(componentFile.getName()).getSequenceEnd();
+ if (fileStart > validComponentSequence || fileEnd > validComponentSequence) {
+ LOGGER.warn(() -> "Deleting invalid component file " + componentFile.getAbsolutePath()
+ + " based on valid sequence " + validComponentSequence);
Files.delete(componentFile.toPath());
}
}