You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2016/11/01 19:47:52 UTC
[14/50] hbase git commit: HBASE-16464 archive folder grows bigger and
bigger due to corrupt snapshot under tmp dir
HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ab011391
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ab011391
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ab011391
Branch: refs/heads/0.98
Commit: ab011391ab392f1a62b6ea9bdca87fc950af42a9
Parents: 3c4e8c1
Author: chenheng <ch...@apache.org>
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Aug 24 18:12:48 2016 -0700
----------------------------------------------------------------------
.../master/snapshot/SnapshotFileCache.java | 17 ++++++++++++++++-
.../master/snapshot/TakeSnapshotHandler.java | 2 +-
.../snapshot/SnapshotDescriptionUtils.java | 14 ++++++++++++++
.../hbase/snapshot/SnapshotManifestV2.java | 18 ++++++++++++------
.../snapshot/TestSnapshotHFileCleaner.java | 20 ++++++++++++++++++++
.../hbase/snapshot/SnapshotTestingUtils.java | 12 ++++++++++++
6 files changed, 75 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab011391/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
- snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+ try {
+ snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+ } catch (CorruptedSnapshotException e) {
+ // See HBASE-16464
+ if (e.getCause() instanceof FileNotFoundException) {
+ // If the snapshot is not in progress, we will delete it
+ if (!fs.exists(new Path(run.getPath(),
+ SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+ fs.delete(run.getPath(), true);
+ LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
+ }
+ } else {
+ throw e;
+ }
+ }
}
}
return snapshotInProgress;
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab011391/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 806975a..acd8161 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -161,7 +161,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot should fail, triggering
// an external exception that gets captured here.
-
+ SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab011391/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index fc5037a..ef28ccc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.snapshot;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
@@ -105,6 +106,9 @@ public final class SnapshotDescriptionUtils {
/** Temporary directory under the snapshot directory to store in-progress snapshots */
public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
+
+ /** This tag will be created in in-progess snapshots */
+ public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
// snapshot operation values
/** Default value if no start time is specified */
public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
@@ -303,6 +307,16 @@ public final class SnapshotDescriptionUtils {
}
/**
+ * Create in-progress tag under .tmp of in-progress snapshot
+ * */
+ public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException {
+ FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
+ HConstants.DATA_FILE_UMASK_KEY);
+ Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
+ FSUtils.create(fs, snapshot_in_progress, perms, true);
+ }
+
+ /**
* Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
* @param fs filesystem where the snapshot was taken
* @param snapshotDir directory where the snapshot was stored
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab011391/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
index 84bc1be..b8362a3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
@@ -81,12 +81,18 @@ public class SnapshotManifestV2 {
}
public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
- SnapshotRegionManifest manifest = region.build();
- FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
- try {
- manifest.writeTo(stream);
- } finally {
- stream.close();
+ // we should ensure the snapshot dir exist, maybe it has been deleted by master
+ // see HBASE-16464
+ if (fs.exists(snapshotDir)) {
+ SnapshotRegionManifest manifest = region.build();
+ FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
+ try {
+ manifest.writeTo(stream);
+ } finally {
+ stream.close();
+ }
+ } else {
+ LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab011391/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
index 4488f86..d20b28c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
@@ -166,4 +166,24 @@ public class TestSnapshotHFileCleaner {
fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
}
}
+
+
+ /**
+ * HBASE-16464
+ */
+ @Test
+ public void testMissedTmpSnapshot() throws IOException {
+ SnapshotTestingUtils.SnapshotMock
+ snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+ SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
+ SNAPSHOT_NAME_STR, TABLE_NAME_STR);
+ builder.addRegionV2();
+ builder.missOneRegionSnapshotFile();
+
+ long period = Long.MAX_VALUE;
+ SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+ "test-snapshot-file-cache-refresh", new SnapshotFiles());
+ cache.getSnapshotsInProgress();
+ assertFalse(fs.exists(builder.getSnapshotsDir()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ab011391/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index d1b7ba7..9da121d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -574,6 +574,18 @@ public final class SnapshotTestingUtils {
corruptFile(p);
}
+ public void missOneRegionSnapshotFile() throws IOException {
+ FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir);
+ for (FileStatus fileStatus : manifestFiles) {
+ String fileName = fileStatus.getPath().getName();
+ if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
+ || fileName.endsWith(".tabledesc")
+ || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+ fs.delete(fileStatus.getPath(), true);
+ }
+ }
+ }
+
/**
* Corrupt data-manifest file
*