You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org> on 2019/04/07 18:12:08 UTC

Change in asterixdb[stabilization-f69489]: [NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted

Murtadha Hubail has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/3331

Change subject: [NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted
......................................................................

[NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- When cleaning up an index's invalid components, check the
  components end sequence in addition to the start sequence
  to ensure invalid merged components with are cleaned too.
- Add test case.

Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
2 files changed, 22 insertions(+), 16 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/31/3331/1

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
index 54ae683..00d2d3d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.test.storage;
 
 import java.io.File;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -127,19 +128,9 @@
         String invalidComponentRange = invalidComponentId + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
         FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath);
         String indexDir = indexDirRef.getFile().getAbsolutePath();
-        // create the invalid component files
-        Path btreePath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER
-                + AbstractLSMIndexFileManager.BTREE_SUFFIX);
-        Path filterPath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER
-                + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
-        Files.createFile(btreePath);
-        Files.createFile(filterPath);
-
-        // clean up the index partition
-        localResourceRepository.cleanup(lr.getPartition());
-        // ensure that the invalid component was deleted
-        Assert.assertFalse(btreePath.toFile().exists());
-        Assert.assertFalse(filterPath.toFile().exists());
+        ensureInvalidComponentDeleted(indexDir, invalidComponentRange, localResourceRepository, lr);
+        String invalidMergeComponentRange = "0" + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
+        ensureInvalidComponentDeleted(indexDir, invalidMergeComponentRange, localResourceRepository, lr);
 
         // ensure that valid components still exist
         // find index valid component timestamp from checkpoint
@@ -179,4 +170,17 @@
         Assert.assertFalse(indexMetadataFile.exists());
         Assert.assertFalse(indexMetadataMaskFile.exists());
     }
+
+    private void ensureInvalidComponentDeleted(String indexDir, String componentSeq,
+            PersistentLocalResourceRepository localResourceRepository, DatasetLocalResource lr) throws IOException {
+        Path btreePath = Paths.get(indexDir,
+                componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BTREE_SUFFIX);
+        Path filterPath = Paths.get(indexDir,
+                componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
+        Files.createFile(btreePath);
+        Files.createFile(filterPath);
+        localResourceRepository.cleanup(lr.getPartition());
+        Assert.assertFalse(btreePath.toFile().exists());
+        Assert.assertFalse(filterPath.toFile().exists());
+    }
 }
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 8f870c0..aef7bbd 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -497,10 +497,12 @@
         }
         final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence();
         for (File componentFile : indexComponentFiles) {
-            // delete any file with start sequence > valid component sequence
+            // delete any file with start or end sequence > valid component sequence
             final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart();
-            if (fileStart > validComponentSequence) {
-                LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath());
+            final long fileEnd = IndexComponentFileReference.of(componentFile.getName()).getSequenceEnd();
+            if (fileStart > validComponentSequence || fileEnd > validComponentSequence) {
+                LOGGER.warn(() -> "Deleting invalid component file " + componentFile.getAbsolutePath()
+                        + " based on valid sequence " + validComponentSequence);
                 Files.delete(componentFile.toPath());
             }
         }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/3331
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: stabilization-f69489
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>