You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2014/05/02 19:32:05 UTC
svn commit: r1591981 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/util/FSVisitor.java
test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
Author: enis
Date: Fri May 2 17:32:05 2014
New Revision: 1591981
URL: http://svn.apache.org/r1591981
Log:
HBASE-9445 Snapshots should create column family dirs for empty regions
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java?rev=1591981&r1=1591980&r2=1591981&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java Fri May 2 17:32:05 2014
@@ -29,11 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.util.FSUtils;
/**
* Utility methods for interacting with the hbase.root file system.
@@ -42,6 +38,10 @@ import org.apache.hadoop.hbase.util.FSUt
public final class FSVisitor {
private static final Log LOG = LogFactory.getLog(FSVisitor.class);
+ public interface RegionVisitor {
+ void region(final String region) throws IOException;
+ }
+
public interface StoreFileVisitor {
void storeFile(final String region, final String family, final String hfileName)
throws IOException;
@@ -69,6 +69,27 @@ public final class FSVisitor {
* @param visitor callback object to get the store files
* @throws IOException if an error occurred while scanning the directory
*/
+ public static void visitRegions(final FileSystem fs, final Path tableDir,
+ final RegionVisitor visitor) throws IOException {
+ FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
+ if (regions == null) {
+ LOG.info("No regions under directory:" + tableDir);
+ return;
+ }
+
+ for (FileStatus region: regions) {
+ visitor.region(region.getPath().getName());
+ }
+ }
+
+ /**
+ * Iterate over the table store files
+ *
+ * @param fs {@link FileSystem}
+ * @param tableDir {@link Path} to the table directory
+ * @param visitor callback object to get the store files
+ * @throws IOException if an error occurred while scanning the directory
+ */
public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
final StoreFileVisitor visitor) throws IOException {
FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java?rev=1591981&r1=1591980&r2=1591981&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java Fri May 2 17:32:05 2014
@@ -49,6 +49,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.google.common.collect.Lists;
+
/**
* Test create/using/deleting snapshots from the client
* <p>
@@ -252,4 +254,56 @@ public class TestSnapshotFromClient {
LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
}
}
+
+ @Test (timeout=300000)
+ public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
+ // test with an empty table with one region
+
+ HBaseAdmin admin = UTIL.getHBaseAdmin();
+ // make sure we don't fail on listing snapshots
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+
+ // get the name of all the regionservers hosting the snapshotted table
+ Set<String> snapshotServers = new HashSet<String>();
+ List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
+ for (RegionServerThread server : servers) {
+ if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
+ snapshotServers.add(server.getRegionServer().getServerName().toString());
+ }
+ }
+
+ LOG.debug("FS state before disable:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+ admin.disableTable(TABLE_NAME);
+
+ LOG.debug("FS state before snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ // take a snapshot of the disabled table
+ byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
+ admin.snapshot(snapshot, TABLE_NAME);
+ LOG.debug("Snapshot completed.");
+
+ // make sure we have the snapshot
+ List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
+ snapshot, TABLE_NAME);
+
+ // make sure its a valid snapshot
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ LOG.debug("FS state after snapshot:");
+ FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
+ FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+
+ List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
+ List<byte[]> nonEmptyCfs = Lists.newArrayList();
+ SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir,
+ admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+
+ admin.deleteSnapshot(snapshot);
+ snapshots = admin.listSnapshots();
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ }
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1591981&r1=1591980&r2=1591981&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java Fri May 2 17:32:05 2014
@@ -204,10 +204,16 @@ public class SnapshotTestingUtils {
// Extract regions and families with store files
final Set<String> snapshotRegions = new HashSet<String>();
final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ FSVisitor.visitRegions(fs, snapshotDir, new FSVisitor.RegionVisitor() {
+ @Override
+ public void region(String region) throws IOException {
+ snapshotRegions.add(region);
+ }
+ });
FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+ @Override
public void storeFile(final String region, final String family, final String hfileName)
throws IOException {
- snapshotRegions.add(region);
snapshotFamilies.add(Bytes.toBytes(family));
}
});
@@ -226,13 +232,6 @@ public class SnapshotTestingUtils {
}
}
- // Avoid checking regions if the request is for an empty snapshot
- if ((nonEmptyTestFamilies == null || nonEmptyTestFamilies.size() == 0) &&
- (emptyTestFamilies != null && emptyTestFamilies.size() > 0)) {
- assertEquals(0, snapshotRegions.size());
- return;
- }
-
// check the region snapshot for all the regions
List<HRegionInfo> regions = admin.getTableRegions(tableName);
assertEquals(regions.size(), snapshotRegions.size());
@@ -343,6 +342,7 @@ public class SnapshotTestingUtils {
throws IOException {
final ArrayList<Path> hfiles = new ArrayList<Path>();
FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
+ @Override
public void storeFile(final String region, final String family, final String hfileName)
throws IOException {
hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
@@ -415,6 +415,7 @@ public class SnapshotTestingUtils {
final ArrayList corruptedFiles = new ArrayList();
SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+ @Override
public void storeFile (final String region, final String family, final String hfile)
throws IOException {
HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);