You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2021/07/30 16:43:35 UTC
[accumulo] branch main updated: Handle old sorted map files in
Upgrade (#2185)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 09e3d77 Handle old sorted map files in Upgrade (#2185)
09e3d77 is described below
commit 09e3d771ef11fcc2c2a8eb1d1b29b024d7da900f
Author: Mike Miller <mm...@apache.org>
AuthorDate: Fri Jul 30 12:43:30 2021 -0400
Handle old sorted map files in Upgrade (#2185)
* Add dropSortedMapWALFiles to Upgrader9to10 for resolving sorted
map files that may still be around during upgrade. Follow up for #2117
* Closes #2179
---
.../accumulo/manager/upgrade/Upgrader9to10.java | 41 +++++++++++++++++++++
.../manager/upgrade/Upgrader9to10Test.java | 42 ++++++++++++++++++++++
2 files changed, 83 insertions(+)
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
index 5e874c5..597f88c 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
@@ -105,6 +105,12 @@ import com.google.common.base.Preconditions;
* <a href="https://github.com/apache/accumulo/issues/1642">#1642</a>, and
* <a href="https://github.com/apache/accumulo/issues/1643">#1643</a> as well.</li>
* </ul>
+ *
+ * Sorted recovery was updated to use RFiles instead of map files. So to prevent issues during
+ * tablet recovery, remove the old temporary map files and resort using RFiles. This is done in
+ * {@link #dropSortedMapWALFiles(VolumeManager)}. For more information see the following issues:
+ * <a href="https://github.com/apache/accumulo/issues/2117">#2117</a> and
+ * <a href="https://github.com/apache/accumulo/issues/2179">#2179</a>
*/
public class Upgrader9to10 implements Upgrader {
@@ -142,6 +148,8 @@ public class Upgrader9to10 implements Upgrader {
upgradeRelativePaths(ctx, Ample.DataLevel.USER);
upgradeDirColumns(ctx, Ample.DataLevel.USER);
upgradeFileDeletes(ctx, Ample.DataLevel.USER);
+ // special case where old files need to be deleted
+ dropSortedMapWALFiles(ctx.getVolumeManager());
}
private void setMetaTableProps(ServerContext ctx) {
@@ -726,4 +734,37 @@ public class Upgrader9to10 implements Upgrader {
}
return new Path(upgradeProperty, VolumeManager.FileType.TABLE.getDirectory() + oldDelete);
}
+
+ /**
+ * Remove old temporary map files to prevent problems during recovery.
+ */
+ static void dropSortedMapWALFiles(VolumeManager vm) {
+ Path recoveryDir = new Path("/accumulo/recovery");
+ try {
+ if (!vm.exists(recoveryDir)) {
+ log.info("There are no recovery files in /accumulo/recovery");
+ return;
+ }
+ List<Path> directoriesToDrop = new ArrayList<>();
+ for (FileStatus walDir : vm.listStatus(recoveryDir)) {
+ // map files will be in a directory starting with "part"
+ Path walDirPath = walDir.getPath();
+ for (FileStatus dirOrFile : vm.listStatus(walDirPath)) {
+ if (dirOrFile.isDirectory()) {
+ directoriesToDrop.add(walDirPath);
+ break;
+ }
+ }
+ }
+ if (!directoriesToDrop.isEmpty()) {
+ log.info("Found {} old sorted map directories to delete.", directoriesToDrop.size());
+ for (Path dir : directoriesToDrop) {
+ log.info("Deleting everything in old sorted map directory: {}", dir);
+ vm.deleteRecursively(dir);
+ }
+ }
+ } catch (IOException ioe) {
+ throw new UncheckedIOException(ioe);
+ }
+ }
}
diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
index a5752b9..25800da 100644
--- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
+++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
@@ -24,11 +24,13 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -49,6 +51,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Da
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.gc.GcVolumeUtil;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.Test;
@@ -324,4 +327,43 @@ public class Upgrader9to10Test {
assertEquals("Replacements should have update for every delete", deleteCount, updateCount);
}
+
+ @Test
+ public void testDropSortedMapWALs() throws IOException {
+ Path recoveryDir = new Path("/accumulo/recovery");
+ VolumeManager fs = createMock(VolumeManager.class);
+ FileStatus[] dirs = new FileStatus[2];
+ dirs[0] = createMock(FileStatus.class);
+ Path dir0 = new Path("/accumulo/recovery/A123456789");
+ FileStatus[] dir0Files = new FileStatus[1];
+ dir0Files[0] = createMock(FileStatus.class);
+ dirs[1] = createMock(FileStatus.class);
+ Path dir1 = new Path("/accumulo/recovery/B123456789");
+ FileStatus[] dir1Files = new FileStatus[1];
+ dir1Files[0] = createMock(FileStatus.class);
+ Path part1Dir = new Path("/accumulo/recovery/B123456789/part-r-0000");
+
+ expect(fs.exists(recoveryDir)).andReturn(true).once();
+ expect(fs.listStatus(recoveryDir)).andReturn(dirs).once();
+ expect(dirs[0].getPath()).andReturn(dir0).once();
+ expect(fs.listStatus(dir0)).andReturn(dir0Files).once();
+ expect(dir0Files[0].isDirectory()).andReturn(false).once();
+
+ expect(dirs[1].getPath()).andReturn(dir1).once();
+ expect(fs.listStatus(dir1)).andReturn(dir1Files).once();
+ expect(dir1Files[0].isDirectory()).andReturn(true).once();
+ expect(dir1Files[0].getPath()).andReturn(part1Dir).once();
+
+ expect(fs.deleteRecursively(dir1)).andReturn(true).once();
+
+ replay(fs, dirs[0], dirs[1], dir0Files[0], dir1Files[0]);
+ Upgrader9to10.dropSortedMapWALFiles(fs);
+
+ reset(fs);
+
+ // test case where there is no recovery
+ expect(fs.exists(recoveryDir)).andReturn(false).once();
+ replay(fs);
+ Upgrader9to10.dropSortedMapWALFiles(fs);
+ }
}