You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/05/02 10:46:58 UTC
svn commit: r1478300 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Author: mbertozzi
Date: Thu May 2 08:46:58 2013
New Revision: 1478300
URL: http://svn.apache.org/r1478300
Log:
HBASE-8455 Update ExportSnapshot to reflect changes in HBASE-7419
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1478300&r1=1478299&r2=1478300&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Thu May 2 08:46:58 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HLogLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.snapshot.ExportSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
@@ -154,7 +155,7 @@ public final class ExportSnapshot extend
*/
private Path getOutputPath(final Path inputPath) throws IOException {
Path path;
- if (HFileLink.isHFileLink(inputPath)) {
+ if (HFileLink.isHFileLink(inputPath) || StoreFile.isReference(inputPath)) {
String family = inputPath.getParent().getName();
String table = HFileLink.getReferencedTableName(inputPath.getName());
String region = HFileLink.getReferencedRegionName(inputPath.getName());
@@ -183,10 +184,12 @@ public final class ExportSnapshot extend
if (inputStat == null) return false;
// Verify if the output file exists and is the same that we want to copy
- FileStatus outputStat = getFileStatus(outputFs, outputPath);
- if (outputStat != null && sameFile(inputStat, outputStat)) {
- LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
- return true;
+ if (outputFs.exists(outputPath)) {
+ FileStatus outputStat = outputFs.getFileStatus(outputPath);
+ if (sameFile(inputStat, outputStat)) {
+ LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+ return true;
+ }
}
context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
@@ -295,7 +298,7 @@ public final class ExportSnapshot extend
private FSDataInputStream openSourceFile(final Path path) {
try {
- if (HFileLink.isHFileLink(path)) {
+ if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
} else if (isHLogLinkPath(path)) {
String serverName = path.getParent().getName();
@@ -311,7 +314,7 @@ public final class ExportSnapshot extend
private FileStatus getFileStatus(final FileSystem fs, final Path path) {
try {
- if (HFileLink.isHFileLink(path)) {
+ if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
HFileLink link = new HFileLink(inputRoot, inputArchive, path);
return link.getFileStatus(fs);
} else if (isHLogLinkPath(path)) {
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1478300&r1=1478299&r2=1478300&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Thu May 2 08:46:58 2013
@@ -36,6 +36,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -51,6 +53,8 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.mapreduce.Job;
@@ -110,18 +114,14 @@ public class TestExportSnapshot {
admin.createTable(htd, null);
// Take an empty snapshot
- admin.disableTable(tableName);
admin.snapshot(emptySnapshotName, tableName);
- admin.enableTable(tableName);
// Add some rows
HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
TEST_UTIL.loadTable(table, FAMILY);
// take a snapshot
- admin.disableTable(tableName);
admin.snapshot(snapshotName, tableName);
- admin.enableTable(tableName);
}
@After
@@ -184,6 +184,56 @@ public class TestExportSnapshot {
}
/**
+ * Mock a snapshot with files in the archive dir,
+ * two regions, and one reference file.
+ */
+ @Test
+ public void testSnapshotWithRefsExportFileSystemState() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+
+ final byte[] tableWithRefsName = Bytes.toBytes("tableWithRefs");
+ final String snapshotName = "tableWithRefs";
+ final String TEST_FAMILY = Bytes.toString(FAMILY);
+ final String TEST_HFILE = "abc";
+
+ final SnapshotDescription sd = SnapshotDescription.newBuilder()
+ .setName(snapshotName).setTable(Bytes.toString(tableWithRefsName)).build();
+
+ FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+ HTableDescriptor htd = new HTableDescriptor(tableWithRefsName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+ // First region, simple with one plain hfile.
+ HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+ conf, htd, null, true, true);
+ Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE);
+ FSDataOutputStream out = fs.create(storeFile);
+ out.write(Bytes.toBytes("Test Data"));
+ out.close();
+ r0.close();
+
+ // Second region, used to test the split case.
+ // This region contains a reference to the hfile in the first region.
+ HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
+ conf, htd, null, true, true);
+ out = fs.create(new Path(new Path(r1.getRegionDir(), TEST_FAMILY),
+ storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName()));
+ out.write(Bytes.toBytes("Test Data"));
+ out.close();
+ r1.close();
+
+ Path tableDir = HTableDescriptor.getTableDir(archiveDir, tableWithRefsName);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+ FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
+ SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
+
+ testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
+ }
+
+ /**
* Test ExportSnapshot
*/
private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
@@ -214,7 +264,8 @@ public class TestExportSnapshot {
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
fs, new Path(copyDir, snapshotDir));
- verifyArchive(fs, copyDir, Bytes.toString(snapshotName));
+ verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
+ FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
// Remove the exported dir
fs.delete(copyDir, true);
@@ -232,11 +283,11 @@ public class TestExportSnapshot {
/*
* Verify if the files exists
*/
- private void verifyArchive(final FileSystem fs, final Path rootDir, final String snapshotName)
- throws IOException {
+ private void verifyArchive(final FileSystem fs, final Path rootDir,
+ final byte[] tableName, final String snapshotName) throws IOException {
final Path exportedSnapshot = new Path(rootDir,
new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
- final Path exportedArchive = new Path(rootDir, ".archive");
+ final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
new SnapshotReferenceUtil.FileVisitor() {
@@ -258,9 +309,8 @@ public class TestExportSnapshot {
}
private void verifyNonEmptyFile(final Path path) throws IOException {
- LOG.debug(path);
- assertTrue(fs.exists(path));
- assertTrue(fs.getFileStatus(path).getLen() > 0);
+ assertTrue(path + " should exist", fs.exists(path));
+ assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
}
});
}