You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/05/08 19:23:52 UTC
svn commit: r1593337 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Author: mbertozzi
Date: Thu May 8 17:23:52 2014
New Revision: 1593337
URL: http://svn.apache.org/r1593337
Log:
HBASE-11119 Update ExportSnapShot to optionally not use a tmp file on external file system (Ted Malaska)
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1593337&r1=1593336&r2=1593337&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Thu May 8 17:23:52 2014
@@ -93,6 +93,7 @@ public final class ExportSnapshot extend
private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
private static final String CONF_BANDWIDTH_MB = "snapshot.export.map.bandwidth.mb";
+ protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
@@ -751,9 +752,12 @@ public final class ExportSnapshot extend
FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
+ boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
+
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+ Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
// Check if the snapshot already exists
if (outputFs.exists(outputSnapshotDir)) {
@@ -769,18 +773,20 @@ public final class ExportSnapshot extend
}
}
- // Check if the snapshot already in-progress
- if (outputFs.exists(snapshotTmpDir)) {
- if (overwrite) {
- if (!outputFs.delete(snapshotTmpDir, true)) {
- System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
+ if (!skipTmp) {
+ // Check if the snapshot already in-progress
+ if (outputFs.exists(snapshotTmpDir)) {
+ if (overwrite) {
+ if (!outputFs.delete(snapshotTmpDir, true)) {
+ System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
+ return 1;
+ }
+ } else {
+ System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
+ System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
+ System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
return 1;
}
- } else {
- System.err.println("A snapshot with the same name '"+ snapshotName +"' may be in-progress");
- System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
- System.err.println("consider removing "+ snapshotTmpDir +" by using the -overwrite option");
- return 1;
}
}
@@ -797,10 +803,10 @@ public final class ExportSnapshot extend
// will remove them because they are unreferenced.
try {
LOG.info("Copy Snapshot Manifest");
- FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+ FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
} catch (IOException e) {
throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
- snapshotDir + " to=" + snapshotTmpDir, e);
+ snapshotDir + " to=" + initialOutputSnapshotDir, e);
}
// Step 2 - Start MR Job to copy files
@@ -814,11 +820,14 @@ public final class ExportSnapshot extend
filesUser, filesGroup, filesMode, mappers, bandwidthMB);
}
- // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
+
LOG.info("Finalize the Snapshot Export");
- if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
- throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
- snapshotTmpDir + " to=" + outputSnapshotDir);
+ if (!skipTmp) {
+ // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
+ if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
+ throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
+ snapshotTmpDir + " to=" + outputSnapshotDir);
+ }
}
// Step 4 - Verify snapshot validity
@@ -829,7 +838,9 @@ public final class ExportSnapshot extend
return 0;
} catch (Exception e) {
LOG.error("Snapshot export failed", e);
- outputFs.delete(snapshotTmpDir, true);
+ if (!skipTmp) {
+ outputFs.delete(snapshotTmpDir, true);
+ }
outputFs.delete(outputSnapshotDir, true);
return 1;
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1593337&r1=1593336&r2=1593337&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Thu May 8 17:23:52 2014
@@ -191,6 +191,12 @@ public class TestExportSnapshot {
}
@Test
+ public void testExportFileSystemStateWithSkipTmp() throws Exception {
+ TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
+ testExportFileSystemState(tableName, snapshotName, 2);
+ }
+
+ @Test
public void testEmptyExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, emptySnapshotName, 1);
}