You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/05/08 19:24:34 UTC
svn commit: r1593340 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Author: mbertozzi
Date: Thu May 8 17:24:33 2014
New Revision: 1593340
URL: http://svn.apache.org/r1593340
Log:
HBASE-11119 Update ExportSnapShot to optionally not use a tmp file on external file system (Ted Malaska)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1593340&r1=1593339&r2=1593340&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Thu May 8 17:24:33 2014
@@ -88,6 +88,7 @@ public final class ExportSnapshot extend
private static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
+ protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
@@ -691,9 +692,12 @@ public final class ExportSnapshot extend
FileSystem inputFs = FileSystem.get(conf);
FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
+ boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
+
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+ Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
// Check if the snapshot already exists
if (outputFs.exists(outputSnapshotDir)) {
@@ -710,17 +714,20 @@ public final class ExportSnapshot extend
}
// Check if the snapshot already in-progress
- if (outputFs.exists(snapshotTmpDir)) {
- if (overwrite) {
- if (!outputFs.delete(snapshotTmpDir, true)) {
- System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
+ if (!skipTmp) {
+ // Check if the snapshot already in-progress
+ if (outputFs.exists(snapshotTmpDir)) {
+ if (overwrite) {
+ if (!outputFs.delete(snapshotTmpDir, true)) {
+ System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
+ return 1;
+ }
+ } else {
+ System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
+ System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
+ System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
return 1;
}
- } else {
- System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
- System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
- System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
- return 1;
}
}
@@ -735,10 +742,11 @@ public final class ExportSnapshot extend
// The snapshot references must be copied before the hfiles otherwise the cleaner
// will remove them because they are unreferenced.
try {
- FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+ LOG.info("Copy Snapshot Manifest");
+ FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
} catch (IOException e) {
throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
- snapshotDir + " to=" + snapshotTmpDir);
+ snapshotDir + " to=" + initialOutputSnapshotDir, e);
}
// Step 2 - Start MR Job to copy files
@@ -753,14 +761,19 @@ public final class ExportSnapshot extend
}
// Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
- if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
- throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
- snapshotTmpDir + " to=" + outputSnapshotDir);
+ if (!skipTmp) {
+ // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
+ if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
+ throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
+ snapshotTmpDir + " to=" + outputSnapshotDir);
+ }
}
return 0;
} catch (Exception e) {
LOG.error("Snapshot export failed", e);
- outputFs.delete(snapshotTmpDir, true);
+ if (!skipTmp) {
+ outputFs.delete(snapshotTmpDir, true);
+ }
outputFs.delete(outputSnapshotDir, true);
return 1;
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1593340&r1=1593339&r2=1593340&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Thu May 8 17:24:33 2014
@@ -179,6 +179,12 @@ public class TestExportSnapshot {
}
@Test
+ public void testExportFileSystemStateWithSkipTmp() throws Exception {
+ TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
+ testExportFileSystemState(tableName, snapshotName, 2);
+ }
+
+ @Test
public void testEmptyExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, emptySnapshotName, 1);
}