You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/02/21 11:01:59 UTC
svn commit: r1570506 - in /hbase/branches/0.96/hbase-server/src:
main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Author: mbertozzi
Date: Fri Feb 21 10:01:59 2014
New Revision: 1570506
URL: http://svn.apache.org/r1570506
Log:
HBASE-10567 Add overwrite manifest option to ExportSnapshot
Modified:
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1570506&r1=1570505&r2=1570506&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Fri Feb 21 10:01:59 2014
@@ -189,7 +189,7 @@ public final class ExportSnapshot extend
// FLAKY-TEST-WARN: lower is better, we can get some runs without the
// retry, but at least we reduce the number of test failures due to
// this test exception from the same map task.
- if (random.nextFloat() < 0.05) {
+ if (random.nextFloat() < 0.03) {
throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath
+ " time=" + System.currentTimeMillis());
}
@@ -587,6 +587,7 @@ public final class ExportSnapshot extend
public int run(String[] args) throws Exception {
boolean verifyChecksum = true;
String snapshotName = null;
+ boolean overwrite = false;
String filesGroup = null;
String filesUser = null;
Path outputRoot = null;
@@ -611,6 +612,8 @@ public final class ExportSnapshot extend
filesGroup = args[++i];
} else if (cmd.equals("-chmod")) {
filesMode = Integer.parseInt(args[++i], 8);
+ } else if (cmd.equals("-overwrite")) {
+ overwrite = true;
} else if (cmd.equals("-h") || cmd.equals("--help")) {
printUsageAndExit();
} else {
@@ -644,13 +647,20 @@ public final class ExportSnapshot extend
// Check if the snapshot already exists
if (outputFs.exists(outputSnapshotDir)) {
- System.err.println("The snapshot '" + snapshotName +
- "' already exists in the destination: " + outputSnapshotDir);
- return 1;
+ if (overwrite) {
+ if (!outputFs.delete(outputSnapshotDir, true)) {
+ System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
+ return 1;
+ }
+ } else {
+ System.err.println("The snapshot '" + snapshotName +
+ "' already exists in the destination: " + outputSnapshotDir);
+ return 1;
+ }
}
// Check if the snapshot already in-progress
- if (outputFs.exists(snapshotTmpDir)) {
+ if (!overwrite && outputFs.exists(snapshotTmpDir)) {
System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
@@ -664,7 +674,7 @@ public final class ExportSnapshot extend
// The snapshot references must be copied before the hfiles otherwise the cleaner
// will remove them because they are unreferenced.
try {
- FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+ FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, overwrite, conf);
} catch (IOException e) {
System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
" to=" + snapshotTmpDir);
@@ -710,14 +720,15 @@ public final class ExportSnapshot extend
System.err.println(" -snapshot NAME Snapshot to restore.");
System.err.println(" -copy-to NAME Remote destination hdfs://");
System.err.println(" -no-checksum-verify Do not verify checksum.");
+ System.err.println(" -overwrite Rewrite the snapshot manifest if already exists");
System.err.println(" -chuser USERNAME Change the owner of the files to the specified one.");
System.err.println(" -chgroup GROUP Change the group of the files to the specified one.");
System.err.println(" -chmod MODE Change the permission of the files to the specified one.");
System.err.println(" -mappers Number of mappers to use during the copy (mapreduce.job.maps).");
System.err.println();
System.err.println("Examples:");
- System.err.println(" hbase " + getClass() + " \\");
- System.err.println(" -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\");
+ System.err.println(" hbase " + getClass().getName() + " \\");
+ System.err.println(" -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
System.err.println(" -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
System.exit(1);
}
Modified: hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1570506&r1=1570505&r2=1570506&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Fri Feb 21 10:01:59 2014
@@ -82,6 +82,8 @@ public class TestExportSnapshot {
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
+ TEST_UTIL.getConfiguration().setInt("mapreduce.map.max.attempts", 10);
+ TEST_UTIL.getConfiguration().setInt("mapred.map.max.attempts", 10);
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@@ -176,6 +178,14 @@ public class TestExportSnapshot {
testExportFileSystemState(tableName, emptySnapshotName, 1);
}
+ @Test
+ public void testConsecutiveExports() throws Exception {
+ Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+ testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
+ testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
+ removeExportDir(copyDir);
+ }
+
/**
* Mock a snapshot with files in the archive dir,
* two regions, and one reference file.
@@ -227,21 +237,32 @@ public class TestExportSnapshot {
testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
}
+ private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
+ int filesExpected) throws Exception {
+ Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+ testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
+ removeExportDir(copyDir);
+ }
+
/**
* Test ExportSnapshot
*/
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
- int filesExpected) throws Exception {
- Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+ int filesExpected, Path copyDir, boolean overwrite) throws Exception {
URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
copyDir = copyDir.makeQualified(fs);
+ List<String> opts = new ArrayList<String>();
+ opts.add("-snapshot");
+ opts.add(Bytes.toString(snapshotName));
+ opts.add("-copy-to");
+ opts.add(copyDir.toString());
+ if (overwrite) opts.add("-overwrite");
+
// Export Snapshot
- int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
- "-snapshot", Bytes.toString(snapshotName),
- "-copy-to", copyDir.toString()
- });
+ int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(),
+ opts.toArray(new String[opts.size()]));
assertEquals(0, res);
// Verify File-System state
@@ -261,9 +282,6 @@ public class TestExportSnapshot {
fs, new Path(copyDir, snapshotDir));
verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
-
- // Remove the exported dir
- fs.delete(copyDir, true);
}
/**
@@ -366,4 +384,10 @@ public class TestExportSnapshot {
}
return files;
}
+
+ private void removeExportDir(final Path path) throws IOException {
+ FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+ FSUtils.logFileSystemState(fs, path, LOG);
+ fs.delete(path, true);
+ }
}