You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/05/02 10:27:17 UTC
svn commit: r1478298 - in /hbase/branches/0.95/hbase-server/src:
main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Author: mbertozzi
Date: Thu May 2 08:27:16 2013
New Revision: 1478298
URL: http://svn.apache.org/r1478298
Log:
HBASE-8455 Update ExportSnapshot to reflect changes in HBASE-7419
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1478298&r1=1478297&r2=1478298&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Thu May 2 08:27:16 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.io.HFileL
import org.apache.hadoop.hbase.io.HLogLink;
import org.apache.hadoop.hbase.mapreduce.JobUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -147,7 +148,7 @@ public final class ExportSnapshot extend
*/
private Path getOutputPath(final Path inputPath) throws IOException {
Path path;
- if (HFileLink.isHFileLink(inputPath)) {
+ if (HFileLink.isHFileLink(inputPath) || StoreFileInfo.isReference(inputPath)) {
String family = inputPath.getParent().getName();
String table = HFileLink.getReferencedTableName(inputPath.getName());
String region = HFileLink.getReferencedRegionName(inputPath.getName());
@@ -176,10 +177,12 @@ public final class ExportSnapshot extend
if (inputStat == null) return false;
// Verify if the output file exists and is the same that we want to copy
- FileStatus outputStat = getFileStatus(outputFs, outputPath);
- if (outputStat != null && sameFile(inputStat, outputStat)) {
- LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
- return true;
+ if (outputFs.exists(outputPath)) {
+ FileStatus outputStat = outputFs.getFileStatus(outputPath);
+ if (sameFile(inputStat, outputStat)) {
+ LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+ return true;
+ }
}
context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
@@ -288,7 +291,7 @@ public final class ExportSnapshot extend
private FSDataInputStream openSourceFile(final Path path) {
try {
- if (HFileLink.isHFileLink(path)) {
+ if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
} else if (isHLogLinkPath(path)) {
String serverName = path.getParent().getName();
@@ -304,7 +307,7 @@ public final class ExportSnapshot extend
private FileStatus getFileStatus(final FileSystem fs, final Path path) {
try {
- if (HFileLink.isHFileLink(path)) {
+ if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
HFileLink link = new HFileLink(inputRoot, inputArchive, path);
return link.getFileStatus(fs);
} else if (isHLogLinkPath(path)) {
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1478298&r1=1478297&r2=1478298&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Thu May 2 08:27:16 2013
@@ -36,6 +36,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -51,6 +53,8 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.mapreduce.Job;
@@ -110,18 +114,14 @@ public class TestExportSnapshot {
admin.createTable(htd, null);
// Take an empty snapshot
- admin.disableTable(tableName);
admin.snapshot(emptySnapshotName, tableName);
- admin.enableTable(tableName);
// Add some rows
HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
TEST_UTIL.loadTable(table, FAMILY);
// take a snapshot
- admin.disableTable(tableName);
admin.snapshot(snapshotName, tableName);
- admin.enableTable(tableName);
}
@After
@@ -184,6 +184,55 @@ public class TestExportSnapshot {
}
/**
+ * Mock a snapshot with files in the archive dir,
+ * two regions, and one reference file.
+ */
+ @Test
+ public void testSnapshotWithRefsExportFileSystemState() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+
+ final byte[] tableWithRefsName = Bytes.toBytes("tableWithRefs");
+ final String snapshotName = "tableWithRefs";
+ final String TEST_FAMILY = Bytes.toString(FAMILY);
+ final String TEST_HFILE = "abc";
+
+ final SnapshotDescription sd = SnapshotDescription.newBuilder()
+ .setName(snapshotName).setTable(Bytes.toString(tableWithRefsName)).build();
+
+ FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+
+ // First region, simple with one plain hfile.
+ HRegionInfo hri = new HRegionInfo(tableWithRefsName);
+ HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
+ fs, new Path(archiveDir, hri.getTableNameAsString()), hri);
+ Path storeFile = new Path(rootDir, TEST_HFILE);
+ FSDataOutputStream out = fs.create(storeFile);
+ out.write(Bytes.toBytes("Test Data"));
+ out.close();
+ r0fs.commitStoreFile(TEST_FAMILY, storeFile);
+
+ // Second region, used to test the split case.
+ // This region contains a reference to the hfile in the first region.
+ hri = new HRegionInfo(tableWithRefsName);
+ HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
+ fs, new Path(archiveDir, hri.getTableNameAsString()), hri);
+ storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
+ out = fs.create(storeFile);
+ out.write(Bytes.toBytes("Test Data"));
+ out.close();
+ r1fs.commitStoreFile(TEST_FAMILY, storeFile);
+
+ Path tableDir = HTableDescriptor.getTableDir(archiveDir, tableWithRefsName);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+ FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
+ SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
+
+ testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
+ }
+
+ /**
* Test ExportSnapshot
*/
private void testExportFileSystemState(final byte[] tableName, final byte[] snapshotName,
@@ -214,7 +263,8 @@ public class TestExportSnapshot {
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
fs, new Path(copyDir, snapshotDir));
- verifyArchive(fs, copyDir, Bytes.toString(snapshotName));
+ verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
+ FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
// Remove the exported dir
fs.delete(copyDir, true);
@@ -232,11 +282,11 @@ public class TestExportSnapshot {
/*
* Verify if the files exists
*/
- private void verifyArchive(final FileSystem fs, final Path rootDir, final String snapshotName)
- throws IOException {
+ private void verifyArchive(final FileSystem fs, final Path rootDir,
+ final byte[] tableName, final String snapshotName) throws IOException {
final Path exportedSnapshot = new Path(rootDir,
new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
- final Path exportedArchive = new Path(rootDir, ".archive");
+ final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
new SnapshotReferenceUtil.FileVisitor() {
@@ -258,9 +308,8 @@ public class TestExportSnapshot {
}
private void verifyNonEmptyFile(final Path path) throws IOException {
- LOG.debug(path);
- assertTrue(fs.exists(path));
- assertTrue(fs.getFileStatus(path).getLen() > 0);
+ assertTrue(path + " should exists", fs.exists(path));
+ assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
}
});
}