You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/05/11 13:40:09 UTC
svn commit: r1593778 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Author: mbertozzi
Date: Sun May 11 11:40:09 2014
New Revision: 1593778
URL: http://svn.apache.org/r1593778
Log:
HBASE-11128 Add -target option to ExportSnapshot to export with a different name
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1593778&r1=1593777&r2=1593778&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Sun May 11 11:40:09 2014
@@ -357,7 +357,7 @@ public final class ExportSnapshot extend
/**
* Try to open the "source" file.
- * Throws an IOException if the communication with the inputFs fail or
+ * Throws an IOException if the communication with the inputFs fail or
* if the file is not found.
*/
private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException {
@@ -638,6 +638,7 @@ public final class ExportSnapshot extend
public int run(String[] args) throws IOException {
boolean verifyChecksum = true;
String snapshotName = null;
+ String targetName = null;
boolean overwrite = false;
String filesGroup = null;
String filesUser = null;
@@ -651,6 +652,8 @@ public final class ExportSnapshot extend
try {
if (cmd.equals("-snapshot")) {
snapshotName = args[++i];
+ } else if (cmd.equals("-target")) {
+ targetName = args[++i];
} else if (cmd.equals("-copy-to")) {
outputRoot = new Path(args[++i]);
} else if (cmd.equals("-no-checksum-verify")) {
@@ -687,6 +690,10 @@ public final class ExportSnapshot extend
printUsageAndExit();
}
+ if (targetName == null) {
+ targetName = snapshotName;
+ }
+
Configuration conf = getConf();
Path inputRoot = FSUtils.getRootDir(conf);
FileSystem inputFs = FileSystem.get(conf);
@@ -695,8 +702,8 @@ public final class ExportSnapshot extend
boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
- Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
- Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+ Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
+ Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
// Check if the snapshot already exists
@@ -707,7 +714,7 @@ public final class ExportSnapshot extend
return 1;
}
} else {
- System.err.println("The snapshot '" + snapshotName +
+ System.err.println("The snapshot '" + targetName +
"' already exists in the destination: " + outputSnapshotDir);
return 1;
}
@@ -723,7 +730,7 @@ public final class ExportSnapshot extend
return 1;
}
} else {
- System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
+ System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");
System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
return 1;
@@ -749,6 +756,16 @@ public final class ExportSnapshot extend
snapshotDir + " to=" + initialOutputSnapshotDir, e);
}
+ // Write a new .snapshotinfo if the target name is different from the source name
+ if (!targetName.equals(snapshotName)) {
+ SnapshotDescription snapshotDesc =
+ SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)
+ .toBuilder()
+ .setName(targetName)
+ .build();
+ SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, snapshotTmpDir, outputFs);
+ }
+
// Step 2 - Start MR Job to copy files
// The snapshot references must be copied before the files otherwise the files gets removed
// by the HFileArchiver, since they have no references.
@@ -768,6 +785,8 @@ public final class ExportSnapshot extend
snapshotTmpDir + " to=" + outputSnapshotDir);
}
}
+
+ LOG.info("Export Completed: " + targetName);
return 0;
} catch (Exception e) {
LOG.error("Snapshot export failed", e);
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1593778&r1=1593777&r2=1593778&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Sun May 11 11:40:09 2014
@@ -118,7 +118,7 @@ public class TestExportSnapshot {
// Add some rows
HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
- SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 1000, FAMILY);
+ SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
// take a snapshot
admin.snapshot(snapshotName, tableName);
@@ -175,28 +175,34 @@ public class TestExportSnapshot {
*/
@Test
public void testExportFileSystemState() throws Exception {
- testExportFileSystemState(tableName, snapshotName, 2);
+ testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
}
@Test
public void testExportFileSystemStateWithSkipTmp() throws Exception {
TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
- testExportFileSystemState(tableName, snapshotName, 2);
+ testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
}
@Test
public void testEmptyExportFileSystemState() throws Exception {
- testExportFileSystemState(tableName, emptySnapshotName, 1);
+ testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 1);
}
@Test
public void testConsecutiveExports() throws Exception {
- Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
- testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
- testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
+ Path copyDir = getLocalDestinationDir();
+ testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, false);
+ testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, true);
removeExportDir(copyDir);
}
+ @Test
+ public void testExportWithTargetName() throws Exception {
+ final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
+ testExportFileSystemState(tableName, snapshotName, targetName, 2);
+ }
+
/**
* Mock a snapshot with files in the archive dir,
* two regions, and one reference file.
@@ -244,13 +250,14 @@ public class TestExportSnapshot {
FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
- testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
+ byte[] name = Bytes.toBytes(snapshotName);
+ testExportFileSystemState(tableWithRefsName, name, name, 2);
}
private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
- int filesExpected) throws Exception {
- Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
- testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
+ final byte[] targetName, int filesExpected) throws Exception {
+ Path copyDir = getHdfsDestinationDir();
+ testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false);
removeExportDir(copyDir);
}
@@ -258,7 +265,8 @@ public class TestExportSnapshot {
* Test ExportSnapshot
*/
private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
- int filesExpected, Path copyDir, boolean overwrite) throws Exception {
+ final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite)
+ throws Exception {
URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
copyDir = copyDir.makeQualified(fs);
@@ -268,6 +276,10 @@ public class TestExportSnapshot {
opts.add(Bytes.toString(snapshotName));
opts.add("-copy-to");
opts.add(copyDir.toString());
+ if (targetName != snapshotName) {
+ opts.add("-target");
+ opts.add(Bytes.toString(targetName));
+ }
if (overwrite) opts.add("-overwrite");
// Export Snapshot
@@ -287,9 +299,10 @@ public class TestExportSnapshot {
// compare the snapshot metadata and verify the hfiles
final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
+ final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
- fs, new Path(copyDir, snapshotDir));
- verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
+ fs, new Path(copyDir, targetDir));
+ verifyArchive(fs, copyDir, tableName, Bytes.toString(targetName));
FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
}
@@ -365,6 +378,11 @@ public class TestExportSnapshot {
assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
}
});
+
+ // Verify Snapshot description
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
+ assertTrue(desc.getName().equals(snapshotName));
+ assertTrue(desc.getTable().equals(Bytes.toString(tableName)));
}
private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
@@ -385,6 +403,19 @@ public class TestExportSnapshot {
return files;
}
+ private Path getHdfsDestinationDir() {
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
+ LOG.info("HDFS export destination path: " + path);
+ return path;
+ }
+
+ private Path getLocalDestinationDir() {
+ Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis());
+ LOG.info("Local export destination path: " + path);
+ return path;
+ }
+
private void removeExportDir(final Path path) throws IOException {
FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
FSUtils.logFileSystemState(fs, path, LOG);