You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by do...@apache.org on 2021/10/05 18:41:49 UTC
[accumulo] branch main updated: Fix chops bugs (#2235)
This is an automated email from the ASF dual-hosted git repository.
domgarguilo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8d714bb Fix chops bugs (#2235)
8d714bb is described below
commit 8d714bb19bcf3fc4f29818fd16b04901fdeb3a99
Author: Dom G <47...@users.noreply.github.com>
AuthorDate: Tue Oct 5 14:41:42 2021 -0400
Fix chops bugs (#2235)
* Fix chops bugs
* Adds new MARKING state
Co-authored-by: Keith Turner <kt...@apache.org>
---
.../accumulo/tserver/tablet/CompactableImpl.java | 37 +++++++++++++++++-----
.../tablet/CompactableImplFileManagerTest.java | 9 ++++--
2 files changed, 36 insertions(+), 10 deletions(-)
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 228517e..576fc1f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -87,8 +87,8 @@ import com.google.common.collect.Sets;
/**
* This class exists between compaction services and tablets and tracks state related to compactions
* for a tablet. This class was written to mainly contain code related to tracking files, state, and
- * synchronization. All other code was placed in {@link CompactableUtils} inorder to make this class
- * easier to analyze.
+ * synchronization. All other code was placed in {@link CompactableUtils} in order to make this
+ * class easier to analyze.
*/
public class CompactableImpl implements Compactable {
@@ -111,7 +111,7 @@ public class CompactableImpl implements Compactable {
private Set<CompactionServiceId> servicesUsed = new ConcurrentSkipListSet<>();
enum ChopSelectionStatus {
- SELECTING, SELECTED, NOT_ACTIVE
+ SELECTING, SELECTED, NOT_ACTIVE, MARKING
}
// status of special compactions
@@ -337,7 +337,7 @@ public class CompactableImpl implements Compactable {
if (chopStatus == ChopSelectionStatus.SELECTED) {
if (getFilesToChop(allFiles).isEmpty()) {
- chopStatus = ChopSelectionStatus.NOT_ACTIVE;
+ chopStatus = ChopSelectionStatus.MARKING;
completed = true;
}
}
@@ -347,6 +347,11 @@ public class CompactableImpl implements Compactable {
return completed;
}
+ void finishMarkingChop() {
+ Preconditions.checkState(chopStatus == ChopSelectionStatus.MARKING);
+ chopStatus = ChopSelectionStatus.NOT_ACTIVE;
+ }
+
void addChoppedFiles(Collection<StoredTabletFile> files) {
choppedFiles.addAll(files);
}
@@ -447,6 +452,7 @@ public class CompactableImpl implements Compactable {
switch (chopStatus) {
case NOT_ACTIVE:
case SELECTING:
+ case MARKING:
return Set.of();
case SELECTED: {
if (selectStatus == FileSelectionStatus.NEW
@@ -739,11 +745,25 @@ public class CompactableImpl implements Compactable {
boolean completed;
synchronized (this) {
+ if (closed) {
+ // if closed, do not attempt to transition to the MARKING state
+ return;
+ }
+ // when this returns true it means we transitioned to the MARKING state
completed = fileMgr.finishChop(allFiles);
}
if (completed) {
- markChopped();
+ try {
+ markChopped();
+ } finally {
+ synchronized (this) {
+ // transition the state from MARKING to NOT_ACTIVE
+ fileMgr.finishMarkingChop();
+ this.notifyAll();
+ }
+ }
+
TabletLogger.selected(getExtent(), CompactionKind.CHOP, Set.of());
}
}
@@ -1462,11 +1482,12 @@ public class CompactableImpl implements Compactable {
closed = true;
- // wait while internal jobs are running or external compactions are committing, but do not
- // wait on external compactions that are running
+ // wait while internal jobs are running, external compactions are committing or the status of
+ // chops is MARKING, but do not wait on external compactions that are running
while (runningJobs.stream()
.anyMatch(job -> !((CompactionExecutorIdImpl) job.getExecutor()).isExternalId())
- || !externalCompactionsCommitting.isEmpty()) {
+ || !externalCompactionsCommitting.isEmpty()
+ || fileMgr.chopStatus == ChopSelectionStatus.MARKING) {
try {
wait(50);
} catch (InterruptedException e) {
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
index 42e934f..9b8dfcf 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
@@ -25,6 +25,7 @@ import static org.apache.accumulo.core.spi.compaction.CompactionKind.USER;
import static org.apache.accumulo.tserver.tablet.CompactableImplFileManagerTest.TestFileManager.SELECTION_EXPIRATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import java.time.Duration;
@@ -379,10 +380,11 @@ public class CompactableImplFileManagerTest {
tabletFiles = newFiles("C00004.rf", "C00006.rf", "F00003.rf", "F00004.rf");
assertFalse(fileMgr.finishChop(tabletFiles));
assertEquals(ChopSelectionStatus.SELECTED, fileMgr.getChopStatus());
+ assertThrows(IllegalStateException.class, fileMgr::finishMarkingChop);
assertEquals(newFiles("F00004.rf"), fileMgr.getCandidates(tabletFiles, CHOP, false));
- // simulate compacting the last file to chop.. should cause the chop finish
+ // simulate compacting the last file to chop. should cause the chop finish
var job3 = newJob(CHOP, "F00004.rf");
assertTrue(fileMgr.reserveFiles(job3));
fileMgr.completed(job3, newFile("C00007.rf"));
@@ -390,9 +392,12 @@ public class CompactableImplFileManagerTest {
assertTrue(fileMgr.finishChop(tabletFiles));
assertEquals(Set.of(), fileMgr.getCandidates(tabletFiles, CHOP, false));
- assertEquals(ChopSelectionStatus.NOT_ACTIVE, fileMgr.getChopStatus());
+ assertEquals(ChopSelectionStatus.MARKING, fileMgr.getChopStatus());
assertEquals(Set.of(), fileMgr.getCompactingFiles());
+ fileMgr.finishMarkingChop();
+ assertEquals(ChopSelectionStatus.NOT_ACTIVE, fileMgr.getChopStatus());
+
}
private void assertNoCandidates(TestFileManager fileMgr, Set<StoredTabletFile> tabletFiles,