You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/05/07 23:28:13 UTC
svn commit: r1593139 [5/6] - in /hbase/trunk:
hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/
hbase-protocol/src/main/protobuf/
hbase-server/src/main/java/org/apache/hadoop/hbase/client/
hbase-server/src/main/java/org/apache/ha...
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java Wed May 7 21:28:12 2014
@@ -19,24 +19,28 @@
package org.apache.hadoop.hbase.snapshot;
import java.io.IOException;
-import java.io.FileNotFoundException;
+import java.io.InterruptedIOException;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
-import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.FSVisitor;
/**
@@ -44,8 +48,15 @@ import org.apache.hadoop.hbase.util.FSVi
*/
@InterfaceAudience.Private
public final class SnapshotReferenceUtil {
- public interface FileVisitor extends FSVisitor.StoreFileVisitor,
- FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
+ public static final Log LOG = LogFactory.getLog(SnapshotReferenceUtil.class);
+
+ public interface StoreFileVisitor {
+ void storeFile(final HRegionInfo regionInfo, final String familyName,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException;
+ }
+
+ public interface SnapshotVisitor extends StoreFileVisitor,
+ FSVisitor.LogFileVisitor {
}
private SnapshotReferenceUtil() {
@@ -64,80 +75,79 @@ public final class SnapshotReferenceUtil
}
/**
- * Get the snapshotted recovered.edits dir for the specified region.
- *
- * @param snapshotDir directory where the specific snapshot is stored
- * @param regionName name of the region
- * @return path to the recovered.edits directory for the specified region files.
- */
- public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
- return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
- }
-
- /**
- * Get the snapshot recovered.edits file
+ * Iterate over the snapshot store files, restored.edits and logs
*
- * @param snapshotDir directory where the specific snapshot is stored
- * @param regionName name of the region
- * @param logfile name of the edit file
- * @return full path of the log file for the specified region files.
+ * @param conf The current {@link Configuration} instance.
+ * @param fs {@link FileSystem}
+ * @param snapshotDir {@link Path} to the Snapshot directory
+ * @param visitor callback object to get the referenced files
+ * @throws IOException if an error occurred while scanning the directory
*/
- public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
- return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
+ public static void visitReferencedFiles(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotVisitor visitor)
+ throws IOException {
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ visitReferencedFiles(conf, fs, snapshotDir, desc, visitor);
}
/**
* Iterate over the snapshot store files, restored.edits and logs
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
* @param snapshotDir {@link Path} to the Snapshot directory
+ * @param desc the {@link SnapshotDescription} of the snapshot to verify
* @param visitor callback object to get the referenced files
* @throws IOException if an error occurred while scanning the directory
*/
- public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
- final FileVisitor visitor) throws IOException {
- visitTableStoreFiles(fs, snapshotDir, visitor);
- visitRecoveredEdits(fs, snapshotDir, visitor);
+ public static void visitReferencedFiles(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription desc, final SnapshotVisitor visitor)
+ throws IOException {
+ visitTableStoreFiles(conf, fs, snapshotDir, desc, visitor);
visitLogFiles(fs, snapshotDir, visitor);
}
/**
* Iterate over the snapshot store files
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
* @param snapshotDir {@link Path} to the Snapshot directory
+ * @param desc the {@link SnapshotDescription} of the snapshot to verify
* @param visitor callback object to get the store files
* @throws IOException if an error occurred while scanning the directory
*/
- public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
- final FSVisitor.StoreFileVisitor visitor) throws IOException {
- FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
+ public static void visitTableStoreFiles(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor)
+ throws IOException {
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+ if (regionManifests == null || regionManifests.size() == 0) {
+ LOG.debug("No manifest files present: " + snapshotDir);
+ return;
+ }
+
+ for (SnapshotRegionManifest regionManifest: regionManifests) {
+ visitRegionStoreFiles(regionManifest, visitor);
+ }
}
/**
* Iterate over the snapshot store files in the specified region
*
- * @param fs {@link FileSystem}
- * @param regionDir {@link Path} to the Snapshot region directory
+ * @param manifest snapshot manifest to inspect
* @param visitor callback object to get the store files
* @throws IOException if an error occurred while scanning the directory
*/
- public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
- final FSVisitor.StoreFileVisitor visitor) throws IOException {
- FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
- }
-
- /**
- * Iterate over the snapshot recovered.edits
- *
- * @param fs {@link FileSystem}
- * @param snapshotDir {@link Path} to the Snapshot directory
- * @param visitor callback object to get the recovered.edits files
- * @throws IOException if an error occurred while scanning the directory
- */
- public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
- final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
- FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
+ public static void visitRegionStoreFiles(final SnapshotRegionManifest manifest,
+ final StoreFileVisitor visitor) throws IOException {
+ HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo());
+ for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+ String familyName = familyFiles.getFamilyName().toStringUtf8();
+ for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
+ visitor.storeFile(regionInfo, familyName, storeFile);
+ }
+ }
}
/**
@@ -165,85 +175,151 @@ public final class SnapshotReferenceUtil
*/
public static void verifySnapshot(final Configuration conf, final FileSystem fs,
final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
- final TableName table = TableName.valueOf(snapshotDesc.getTable());
- visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- HFileLink link = HFileLink.create(conf, table, region, family, hfile);
- try {
- link.getFileStatus(fs);
- } catch (FileNotFoundException e) {
- throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+ verifySnapshot(conf, fs, manifest);
+ }
+
+ /**
+ * Verify the validity of the snapshot
+ *
+ * @param conf The current {@link Configuration} instance.
+ * @param fs {@link FileSystem}
+ * @param manifest snapshot manifest to inspect
+ * @throws CorruptedSnapshotException if the snapshot is corrupted
+ * @throws IOException if an error occurred while scanning the directory
+ */
+ public static void verifySnapshot(final Configuration conf, final FileSystem fs,
+ final SnapshotManifest manifest) throws IOException {
+ final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
+ final Path snapshotDir = manifest.getSnapshotDir();
+
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+ if (regionManifests == null || regionManifests.size() == 0) {
+ LOG.debug("No manifest files present: " + snapshotDir);
+ return;
+ }
+
+ ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot");
+ final ExecutorCompletionService<Void> completionService =
+ new ExecutorCompletionService<Void>(exec);
+ try {
+ for (final SnapshotRegionManifest regionManifest: regionManifests) {
+ completionService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ visitRegionStoreFiles(regionManifest, new StoreFileVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile);
+ }
+ });
+ return null;
+ }
+ });
+ }
+ try {
+ for (int i = 0; i < regionManifests.size(); ++i) {
+ completionService.take().get();
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof CorruptedSnapshotException) {
+ throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc);
+ } else {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
}
}
- });
+ } finally {
+ exec.shutdown();
+ }
}
/**
- * Returns the set of region names available in the snapshot.
+ * Verify the validity of the snapshot store file
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
- * @param snapshotDir {@link Path} to the Snapshot directory
+ * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
+ * @param snapshot the {@link SnapshotDescription} of the snapshot to verify
+ * @param regionInfo {@link HRegionInfo} of the region that contains the store file
+ * @param family family that contains the store file
+ * @param storeFile the store file to verify
+ * @throws CorruptedSnapshotException if the snapshot is corrupted
* @throws IOException if an error occurred while scanning the directory
- * @return the set of the regions contained in the snapshot
*/
- public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
- throws IOException {
- FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
- if (regionDirs == null) return null;
+ public static void verifyStoreFile(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription snapshot, final HRegionInfo regionInfo,
+ final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String fileName = storeFile.getName();
+
+ Path refPath = null;
+ if (StoreFileInfo.isReference(fileName)) {
+ // If is a reference file check if the parent file is present in the snapshot
+ Path snapshotHFilePath = new Path(new Path(
+ new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName);
+ refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath);
+ if (!fs.exists(refPath)) {
+ throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot);
+ }
+ }
+
+ Path linkPath;
+ if (refPath != null && HFileLink.isHFileLink(refPath)) {
+ linkPath = new Path(family, refPath.getName());
+ } else if (HFileLink.isHFileLink(fileName)) {
+ linkPath = new Path(family, fileName);
+ } else {
+ linkPath = new Path(family, HFileLink.createHFileLinkName(
+ TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(), fileName));
+ }
- Set<String> regions = new HashSet<String>();
- for (FileStatus regionDir: regionDirs) {
- regions.add(regionDir.getPath().getName());
+ // check if the linked file exists (in the archive, or in the table dir)
+ HFileLink link = new HFileLink(conf, linkPath);
+ if (!link.exists(fs)) {
+ throw new CorruptedSnapshotException("Can't find hfile: " + fileName
+ + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath()
+ + ") directory for the primary table.", snapshot);
}
- return regions;
}
/**
- * Get the list of hfiles for the specified snapshot region.
- * NOTE: The current implementation keeps one empty file per HFile in the region.
- * The file name matches the one in the original table, and by reconstructing
- * the path you can quickly jump to the referenced file.
+ * Returns the store file names in the snapshot.
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
- * @param snapshotRegionDir {@link Path} to the Snapshot region directory
- * @return Map of hfiles per family, the key is the family name and values are hfile names
+ * @param snapshotDir {@link Path} to the Snapshot directory
* @throws IOException if an error occurred while scanning the directory
+ * @return the names of hfiles in the specified snaphot
*/
- public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
- final Path snapshotRegionDir) throws IOException {
- final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
-
- visitRegionStoreFiles(fs, snapshotRegionDir,
- new FSVisitor.StoreFileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- List<String> hfiles = familyFiles.get(family);
- if (hfiles == null) {
- hfiles = new LinkedList<String>();
- familyFiles.put(family, hfiles);
- }
- hfiles.add(hfile);
- }
- });
-
- return familyFiles;
+ public static Set<String> getHFileNames(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir) throws IOException {
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ return getHFileNames(conf, fs, snapshotDir, desc);
}
/**
* Returns the store file names in the snapshot.
*
+ * @param conf The current {@link Configuration} instance.
* @param fs {@link FileSystem}
* @param snapshotDir {@link Path} to the Snapshot directory
+ * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to inspect
* @throws IOException if an error occurred while scanning the directory
* @return the names of hfiles in the specified snaphot
*/
- public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
+ private static Set<String> getHFileNames(final Configuration conf, final FileSystem fs,
+ final Path snapshotDir, final SnapshotDescription snapshotDesc)
throws IOException {
final Set<String> names = new HashSet<String>();
- visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
+ visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, new StoreFileVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfile = storeFile.getName();
if (HFileLink.isHFileLink(hfile)) {
names.add(HFileLink.getReferencedHFileName(hfile));
} else {
@@ -266,6 +342,7 @@ public final class SnapshotReferenceUtil
throws IOException {
final Set<String> names = new HashSet<String>();
visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
+ @Override
public void logFile (final String server, final String logfile) throws IOException {
names.add(logfile);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java Wed May 7 21:28:12 2014
@@ -150,7 +150,7 @@ public final class FSVisitor {
final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
FileStatus[] regions = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
if (regions == null) {
- LOG.info("No regions under directory:" + tableDir);
+ LOG.info("No recoveredEdits regions under directory:" + tableDir);
return;
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java Wed May 7 21:28:12 2014
@@ -20,9 +20,7 @@ package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,9 +37,9 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -182,15 +180,6 @@ public class TestSnapshotFromClient {
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
UTIL.loadTable(table, TEST_FAM, false);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before disable:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -203,8 +192,16 @@ public class TestSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
// take a snapshot of the disabled table
- byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
- admin.snapshot(snapshot, TABLE_NAME);
+ final String SNAPSHOT_NAME = "offlineTableSnapshot";
+ byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
+
+ SnapshotDescription desc = SnapshotDescription.newBuilder()
+ .setType(SnapshotDescription.Type.DISABLED)
+ .setTable(STRING_TABLE_NAME)
+ .setName(SNAPSHOT_NAME)
+ .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
+ .build();
+ admin.snapshot(desc);
LOG.debug("Snapshot completed.");
// make sure we have the snapshot
@@ -219,7 +216,7 @@ public class TestSnapshotFromClient {
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
@@ -263,15 +260,6 @@ public class TestSnapshotFromClient {
// make sure we don't fail on listing snapshots
SnapshotTestingUtils.assertNoSnapshots(admin);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before disable:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -299,8 +287,8 @@ public class TestSnapshotFromClient {
List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
List<byte[]> nonEmptyCfs = Lists.newArrayList();
- SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
+ rootDir, admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java Wed May 7 21:28:12 2014
@@ -29,13 +29,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.mapreduce.TestTableSnapshotInputFormat;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -94,7 +94,7 @@ public class TestTableSnapshotScanner {
HTable table = new HTable(util.getConfiguration(), tableName);
util.loadTable(table, FAMILIES);
- Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR));
+ Path rootDir = FSUtils.getRootDir(util.getConfiguration());
FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
@@ -124,7 +124,7 @@ public class TestTableSnapshotScanner {
testScanner(UTIL, "testWithMultiRegion", 20, true);
}
- private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions,
+ private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions,
boolean shutdownCluster) throws Exception {
setupCluster();
TableName tableName = TableName.valueOf("testScanner");
@@ -138,7 +138,7 @@ public class TestTableSnapshotScanner {
Path restoreDir = util.getDataTestDirOnTestFS(snapshotName);
Scan scan = new Scan(bbb, yyy); // limit the scan
- TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir,
+ TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir,
snapshotName, scan);
verifyScanner(scanner, bbb, yyy);
@@ -155,7 +155,7 @@ public class TestTableSnapshotScanner {
private void verifyScanner(ResultScanner scanner, byte[] startRow, byte[] stopRow)
throws IOException, InterruptedException {
- HBaseTestingUtility.SeenRowTracker rowTracker =
+ HBaseTestingUtility.SeenRowTracker rowTracker =
new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
while (true) {
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java Wed May 7 21:28:12 2014
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
@@ -44,12 +43,12 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -59,7 +58,6 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -125,17 +123,17 @@ public class TestTableSnapshotInputForma
Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution));
blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2);
- Assert.assertEquals(Lists.newArrayList("h1", "h2"),
+ Assert.assertEquals(Lists.newArrayList("h1", "h2"),
tsif.getBestLocations(conf, blockDistribution));
blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3);
- Assert.assertEquals(Lists.newArrayList("h2", "h1"),
+ Assert.assertEquals(Lists.newArrayList("h2", "h1"),
tsif.getBestLocations(conf, blockDistribution));
blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6);
blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9);
- Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"),
+ Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"),
tsif.getBestLocations(conf, blockDistribution));
}
@@ -156,7 +154,7 @@ public class TestTableSnapshotInputForma
public static class TestTableSnapshotReducer
extends Reducer<ImmutableBytesWritable, NullWritable, NullWritable, NullWritable> {
- HBaseTestingUtility.SeenRowTracker rowTracker =
+ HBaseTestingUtility.SeenRowTracker rowTracker =
new HBaseTestingUtility.SeenRowTracker(bbb, yyy);
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<NullWritable> values,
@@ -191,7 +189,7 @@ public class TestTableSnapshotInputForma
HTable table = new HTable(util.getConfiguration(), tableName);
util.loadTable(table, FAMILIES);
- Path rootDir = new Path(util.getConfiguration().get(HConstants.HBASE_DIR));
+ Path rootDir = FSUtils.getRootDir(util.getConfiguration());
FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
@@ -250,7 +248,7 @@ public class TestTableSnapshotInputForma
testWithMockedMapReduce(UTIL, "testWithMockedMapReduceMultiRegion", 10, 8);
}
- public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
+ public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits) throws Exception {
setupCluster();
TableName tableName = TableName.valueOf("testWithMockedMapReduce");
@@ -282,7 +280,7 @@ public class TestTableSnapshotInputForma
Assert.assertEquals(expectedNumSplits, splits.size());
- HBaseTestingUtility.SeenRowTracker rowTracker =
+ HBaseTestingUtility.SeenRowTracker rowTracker =
new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);
for (int i = 0; i < splits.size(); i++) {
@@ -293,7 +291,7 @@ public class TestTableSnapshotInputForma
// validate record reader
TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());
- RecordReader<ImmutableBytesWritable, Result> rr =
+ RecordReader<ImmutableBytesWritable, Result> rr =
tsif.createRecordReader(split, taskAttemptContext);
rr.initialize(split, taskAttemptContext);
@@ -311,7 +309,7 @@ public class TestTableSnapshotInputForma
rowTracker.validate();
}
- public static void verifyRowFromMap(ImmutableBytesWritable key, Result result)
+ public static void verifyRowFromMap(ImmutableBytesWritable key, Result result)
throws IOException {
byte[] row = key.get();
CellScanner scanner = result.cellScanner();
@@ -363,7 +361,7 @@ public class TestTableSnapshotInputForma
// this is also called by the IntegrationTestTableSnapshotInputFormat
public static void doTestWithMapReduce(HBaseTestingUtility util, TableName tableName,
- String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
+ String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
boolean shutdownCluster) throws Exception {
//create the table and snapshot
@@ -379,7 +377,7 @@ public class TestTableSnapshotInputForma
Scan scan = new Scan(bbb, yyy); // limit the scan
job.setJarByClass(util.getClass());
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
TestTableSnapshotInputFormat.class);
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Wed May 7 21:28:12 2014
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
@@ -330,19 +332,20 @@ public class TestSnapshotFromMaster {
// get the snapshot files for the table
Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
- Path[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
+ Set<String> snapshotHFiles = SnapshotReferenceUtil.getHFileNames(
+ UTIL.getConfiguration(), fs, snapshotTable);
// check that the files in the archive contain the ones that we need for the snapshot
LOG.debug("Have snapshot hfiles:");
- for (Path file : snapshotHFiles) {
- LOG.debug(file);
+ for (String fileName : snapshotHFiles) {
+ LOG.debug(fileName);
}
// get the archived files for the table
Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, TABLE_NAME);
// and make sure that there is a proper subset
- for (Path file : snapshotHFiles) {
- assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file,
- files.contains(file.getName()));
+ for (String fileName : snapshotHFiles) {
+ assertTrue("Archived hfiles " + files + " is missing snapshot file:" + fileName,
+ files.contains(fileName));
}
// delete the existing snapshot
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java Wed May 7 21:28:12 2014
@@ -22,7 +22,9 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.util.Collection;
+import java.util.ArrayList;
import java.util.HashSet;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,9 +33,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.AfterClass;
@@ -49,6 +52,7 @@ public class TestSnapshotFileCache {
private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static long sequenceId = 0;
private static FileSystem fs;
private static Path rootDir;
@@ -72,86 +76,17 @@ public class TestSnapshotFileCache {
}
@Test(timeout = 10000000)
- public void testLoadAndDelete() throws Exception {
+ public void testLoadAndDelete() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- Path file2 = new Path(family, "file2");
-
- // create two hfiles under the snapshot
- fs.createNewFile(file1);
- fs.createNewFile(file2);
+ createAndTestSnapshotV1(cache, "snapshot1a", false, true);
+ createAndTestSnapshotV1(cache, "snapshot1b", true, true);
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
- // then make sure the cache finds them
- assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
- assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
- String not = "file-shouldn't-be-found";
- assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
-
- // make sure we get a little bit of separation in the modification times
- // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
- Thread.sleep(10);
-
- LOG.debug("Deleting snapshot.");
- // then delete the snapshot and make sure that we can still find the files
- if (!fs.delete(snapshot, true)) {
- throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
- }
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
-
- LOG.debug("Checking to see if file is deleted.");
- assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
- assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
-
- // then trigger a refresh
- cache.triggerCacheRefreshForTesting();
- // and not it shouldn't find those files
- assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
- cache.contains(file1.getName()));
- assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
- cache.contains(file2.getName()));
-
- fs.delete(snapshotDir, true);
- }
-
- @Test
- public void testLoadsTmpDir() throws Exception {
- // don't refresh the cache unless we tell it to
- long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
- SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
- "test-snapshot-file-cache-refresh", new SnapshotFiles());
-
- // create a file in a 'completed' snapshot
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- fs.createNewFile(file1);
-
- // create an 'in progress' snapshot
- SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
- snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
- region = new Path(snapshot, "7e91021");
- family = new Path(region, "fam");
- Path file2 = new Path(family, "file2");
- fs.createNewFile(file2);
-
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
- // then make sure the cache finds both files
- assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
- assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
+ createAndTestSnapshotV2(cache, "snapshot2a", false, true);
+ createAndTestSnapshotV2(cache, "snapshot2b", true, true);
}
@Test
@@ -168,14 +103,14 @@ public class TestSnapshotFileCache {
});
// create a file in a 'completed' snapshot
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
+ SnapshotDescription desc = SnapshotDescription.newBuilder().setName("snapshot").build();
+ Path snapshot = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
+ SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshot, fs);
+ Path file1 = new Path(new Path(new Path(snapshot, "7e91021"), "fam"), "file1");
fs.createNewFile(file1);
// and another file in the logs directory
- Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
+ Path logs = getSnapshotHLogsDir(snapshot, "server");
Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
fs.createNewFile(log);
@@ -187,65 +122,117 @@ public class TestSnapshotFileCache {
assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
}
+ /**
+ * Get the log directory for a specific snapshot
+ * @param snapshotDir directory where the specific snapshot will be store
+ * @param serverName name of the parent regionserver for the log files
+ * @return path to the log home directory for the archive files.
+ */
+ public static Path getSnapshotHLogsDir(Path snapshotDir, String serverName) {
+ return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
+ }
+
@Test
public void testReloadModifiedDirectory() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- Path snapshot = new Path(snapshotDir, "snapshot");
- Path region = new Path(snapshot, "7e91021");
- Path family = new Path(region, "fam");
- Path file1 = new Path(family, "file1");
- Path file2 = new Path(family, "file2");
-
- // create two hfiles under the snapshot
- fs.createNewFile(file1);
- fs.createNewFile(file2);
-
- FSUtils.logFileSystemState(fs, rootDir, LOG);
-
- assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
-
+ createAndTestSnapshotV1(cache, "snapshot1", false, true);
// now delete the snapshot and add a file with a different name
- fs.delete(snapshot, true);
- Path file3 = new Path(family, "new_file");
- fs.createNewFile(file3);
+ createAndTestSnapshotV1(cache, "snapshot1", false, false);
- FSUtils.logFileSystemState(fs, rootDir, LOG);
- assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
+ createAndTestSnapshotV2(cache, "snapshot2", false, true);
+ // now delete the snapshot and add a file with a different name
+ createAndTestSnapshotV2(cache, "snapshot2", false, false);
}
@Test
public void testSnapshotTempDirReload() throws IOException {
long period = Long.MAX_VALUE;
// This doesn't refresh cache until we invoke it explicitly
- Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
- SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
- // Add a new snapshot
- Path snapshot1 = new Path(snapshotDir, "snapshot1");
- Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
- fs.createNewFile(file1);
- assertTrue(cache.contains(file1.getName()));
+ // Add a new non-tmp snapshot
+ createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
+ createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
+
+ // Add a new tmp snapshot
+ createAndTestSnapshotV2(cache, "snapshot1", true, false);
- // Add another snapshot
- Path snapshot2 = new Path(snapshotDir, "snapshot2");
- Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
- fs.createNewFile(file2);
- assertTrue(cache.contains(file2.getName()));
+ // Add another tmp snapshot
+ createAndTestSnapshotV2(cache, "snapshot2", true, false);
}
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
Collection<String> files = new HashSet<String>();
files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
- files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
+ files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir));
return files;
}
};
+
+ private void createAndTestSnapshotV1(final SnapshotFileCache cache, final String name,
+ final boolean tmp, final boolean removeOnExit) throws IOException {
+ SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name);
+ createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+ }
+
+ private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
+ final boolean tmp, final boolean removeOnExit) throws IOException {
+ SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
+ SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name);
+ createAndTestSnapshot(cache, builder, tmp, removeOnExit);
+ }
+
+ private void createAndTestSnapshot(final SnapshotFileCache cache,
+ final SnapshotMock.SnapshotBuilder builder,
+ final boolean tmp, final boolean removeOnExit) throws IOException {
+ List<String> files = new ArrayList<String>();
+ for (int i = 0; i < 3; ++i) {
+ for (Path filePath: builder.addRegion()) {
+ String fileName = filePath.getName();
+ if (tmp) {
+ // We should be able to find all the files while the snapshot creation is in-progress
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+ assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+ }
+ files.add(fileName);
+ }
+ }
+
+ // Finalize the snapshot
+ if (!tmp) {
+ builder.commit();
+ }
+
+ // Make sure that all files are still present
+ for (String fileName: files) {
+ assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+ }
+
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+ if (removeOnExit) {
+ LOG.debug("Deleting snapshot.");
+ fs.delete(builder.getSnapshotsDir(), true);
+ FSUtils.logFileSystemState(fs, rootDir, LOG);
+
+ // The files should be in cache until next refresh
+ for (String fileName: files) {
+ assertTrue("Cache didn't find " + fileName, cache.contains(fileName));
+ }
+
+ // then trigger a refresh
+ cache.triggerCacheRefreshForTesting();
+ // and not it shouldn't find those files
+ for (String fileName: files) {
+ assertFalse("Cache found '" + fileName + "', but it shouldn't have.",
+ cache.contains(fileName));
+ }
+ }
+ }
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java Wed May 7 21:28:12 2014
@@ -22,14 +22,18 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -43,11 +47,13 @@ import org.apache.hadoop.hbase.client.Du
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -56,6 +62,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSVisitor;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.junit.Assert;
@@ -144,14 +151,12 @@ public class SnapshotTestingUtils {
*/
public static void confirmSnapshotValid(
SnapshotDescription snapshotDescriptor, TableName tableName,
- byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
- boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+ byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
throws IOException {
ArrayList nonEmptyTestFamilies = new ArrayList(1);
nonEmptyTestFamilies.add(testFamily);
confirmSnapshotValid(snapshotDescriptor, tableName,
- nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
- logsDir, snapshotServers);
+ nonEmptyTestFamilies, null, rootDir, admin, fs);
}
/**
@@ -159,14 +164,12 @@ public class SnapshotTestingUtils {
*/
public static void confirmEmptySnapshotValid(
SnapshotDescription snapshotDescriptor, TableName tableName,
- byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
- boolean requireLogs, Path logsDir, Set<String> snapshotServers)
+ byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
throws IOException {
ArrayList emptyTestFamilies = new ArrayList(1);
emptyTestFamilies.add(testFamily);
confirmSnapshotValid(snapshotDescriptor, tableName,
- null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
- logsDir, snapshotServers);
+ null, emptyTestFamilies, rootDir, admin, fs);
}
/**
@@ -178,45 +181,31 @@ public class SnapshotTestingUtils {
public static void confirmSnapshotValid(
SnapshotDescription snapshotDescriptor, TableName tableName,
List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
- Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
- Path logsDir, Set<String> snapshotServers) throws IOException {
+ Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException {
+ final Configuration conf = admin.getConfiguration();
+
// check snapshot dir
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotDescriptor, rootDir);
assertTrue(fs.exists(snapshotDir));
- // check snapshot info
- Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
- assertTrue(fs.exists(snapshotinfo));
-
- // check the logs dir
- if (requireLogs) {
- TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
- snapshotServers, snapshotDescriptor, new Path(snapshotDir,
- HConstants.HREGION_LOGDIR_NAME));
- }
-
- // check the table info
- HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
- HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
- assertEquals(desc, snapshotDesc);
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
// Extract regions and families with store files
- final Set<String> snapshotRegions = new HashSet<String>();
final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- FSVisitor.visitRegions(fs, snapshotDir, new FSVisitor.RegionVisitor() {
- @Override
- public void region(String region) throws IOException {
- snapshotRegions.add(region);
- }
- });
- FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
- @Override
- public void storeFile(final String region, final String family, final String hfileName)
- throws IOException {
- snapshotFamilies.add(Bytes.toBytes(family));
- }
- });
+
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
+ Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
+ for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
+ SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
+ new SnapshotReferenceUtil.StoreFileVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ snapshotFamilies.add(Bytes.toBytes(family));
+ }
+ });
+ }
// Verify that there are store files in the specified families
if (nonEmptyTestFamilies != null) {
@@ -234,16 +223,12 @@ public class SnapshotTestingUtils {
// check the region snapshot for all the regions
List<HRegionInfo> regions = admin.getTableRegions(tableName);
- assertEquals(regions.size(), snapshotRegions.size());
+ assertEquals(regions.size(), regionManifests.size());
- // Verify Regions
+ // Verify Regions (redundant check, see MasterSnapshotVerifier)
for (HRegionInfo info : regions) {
String regionName = info.getEncodedName();
- assertTrue(snapshotRegions.contains(regionName));
-
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
- assertEquals(info, snapshotRegionInfo);
+ assertTrue(regionManifests.containsKey(regionName));
}
}
@@ -351,6 +336,17 @@ public class SnapshotTestingUtils {
return hfiles.toArray(new Path[hfiles.size()]);
}
+ public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
+ throws IOException {
+ Path[] files = listHFiles(fs, tableDir);
+ String[] names = new String[files.length];
+ for (int i = 0; i < files.length; ++i) {
+ names[i] = files[i].getName();
+ }
+ Arrays.sort(names);
+ return names;
+ }
+
/**
* Take a snapshot of the specified table and verify that the given family is
* not empty. Note that this will leave the table disabled
@@ -391,8 +387,7 @@ public class SnapshotTestingUtils {
}
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
- emptyFamilyNames, rootDir, admin, fs, false,
- new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
+ emptyFamilyNames, rootDir, admin, fs);
}
/**
@@ -414,11 +409,15 @@ public class SnapshotTestingUtils {
final TableName table = TableName.valueOf(snapshotDesc.getTable());
final ArrayList corruptedFiles = new ArrayList();
- SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
+ final Configuration conf = util.getConfiguration();
+ SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
+ new SnapshotReferenceUtil.StoreFileVisitor() {
@Override
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String region = regionInfo.getEncodedName();
+ String hfile = storeFile.getName();
+ HFileLink link = HFileLink.create(conf, table, region, family, hfile);
if (corruptedFiles.size() % 2 == 0) {
fs.delete(link.getAvailablePath(fs), true);
corruptedFiles.add(hfile);
@@ -431,6 +430,191 @@ public class SnapshotTestingUtils {
}
// ==========================================================================
+ // Snapshot Mock
+ // ==========================================================================
+ public static class SnapshotMock {
+ private final static String TEST_FAMILY = "cf";
+ public final static int TEST_NUM_REGIONS = 4;
+
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path rootDir;
+
+ static class RegionData {
+ public HRegionInfo hri;
+ public Path tableDir;
+ public Path[] files;
+
+ public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
+ this.tableDir = tableDir;
+ this.hri = hri;
+ this.files = new Path[nfiles];
+ }
+ }
+
+ public static class SnapshotBuilder {
+ private final RegionData[] tableRegions;
+ private final SnapshotDescription desc;
+ private final HTableDescriptor htd;
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path rootDir;
+ private Path snapshotDir;
+ private int snapshotted = 0;
+
+ public SnapshotBuilder(final Configuration conf, final FileSystem fs,
+ final Path rootDir, final HTableDescriptor htd,
+ final SnapshotDescription desc, final RegionData[] tableRegions)
+ throws IOException {
+ this.fs = fs;
+ this.conf = conf;
+ this.rootDir = rootDir;
+ this.htd = htd;
+ this.desc = desc;
+ this.tableRegions = tableRegions;
+ this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+ new FSTableDescriptors(conf)
+ .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
+ }
+
+ public HTableDescriptor getTableDescriptor() {
+ return this.htd;
+ }
+
+ public SnapshotDescription getSnapshotDescription() {
+ return this.desc;
+ }
+
+ public Path getSnapshotsDir() {
+ return this.snapshotDir;
+ }
+
+ public Path[] addRegion() throws IOException {
+ return addRegion(desc);
+ }
+
+ public Path[] addRegionV1() throws IOException {
+ return addRegion(desc.toBuilder()
+ .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
+ .build());
+ }
+
+ public Path[] addRegionV2() throws IOException {
+ return addRegion(desc.toBuilder()
+ .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
+ .build());
+ }
+
+ private Path[] addRegion(final SnapshotDescription desc) throws IOException {
+ if (this.snapshotted == tableRegions.length) {
+ throw new UnsupportedOperationException("No more regions in the table");
+ }
+
+ RegionData regionData = tableRegions[this.snapshotted++];
+ ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
+ SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
+ manifest.addRegion(regionData.tableDir, regionData.hri);
+ return regionData.files;
+ }
+
+ public Path commit() throws IOException {
+ ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
+ SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
+ manifest.addTableDescriptor(htd);
+ manifest.consolidate();
+ SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
+ snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
+ return snapshotDir;
+ }
+ }
+
+ public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
+ this.fs = fs;
+ this.conf = conf;
+ this.rootDir = rootDir;
+ }
+
+ public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
+ return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
+ }
+
+ public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
+ return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
+ }
+
+ private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
+ throws IOException {
+ HTableDescriptor htd = createHtd(snapshotName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+
+ RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
+
+ SnapshotDescription desc = SnapshotDescription.newBuilder()
+ .setTable(htd.getNameAsString())
+ .setName(snapshotName)
+ .setVersion(version)
+ .build();
+
+ Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
+ SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
+ return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
+ }
+
+ public HTableDescriptor createHtd(final String tableName) {
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+ return htd;
+ }
+
+ private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
+ throws IOException {
+ Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
+ new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
+
+ assertTrue(nregions % 2 == 0);
+ RegionData[] regions = new RegionData[nregions];
+ for (int i = 0; i < regions.length; i += 2) {
+ byte[] startKey = Bytes.toBytes(0 + i * 2);
+ byte[] endKey = Bytes.toBytes(1 + i * 2);
+
+ // First region, simple with one plain hfile.
+ HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
+ HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+ regions[i] = new RegionData(tableDir, hri, 3);
+ for (int j = 0; j < regions[i].files.length; ++j) {
+ Path storeFile = createStoreFile(rfs.createTempName());
+ regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
+ }
+
+ // Second region, used to test the split case.
+ // This region contains a reference to the hfile in the first region.
+ startKey = Bytes.toBytes(2 + i * 2);
+ endKey = Bytes.toBytes(3 + i * 2);
+ hri = new HRegionInfo(htd.getTableName());
+ rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+ regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
+ for (int j = 0; j < regions[i].files.length; ++j) {
+ String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
+ Path refFile = createStoreFile(new Path(rootDir, refName));
+ regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
+ }
+ }
+ return regions;
+ }
+
+ private Path createStoreFile(final Path storeFile)
+ throws IOException {
+ FSDataOutputStream out = fs.create(storeFile);
+ try {
+ out.write(Bytes.toBytes(storeFile.toString()));
+ } finally {
+ out.close();
+ }
+ return storeFile;
+ }
+ }
+
+ // ==========================================================================
// Table Helpers
// ==========================================================================
public static void waitForTableToBeOnline(final HBaseTestingUtility util,
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java Wed May 7 21:28:12 2014
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -48,9 +47,12 @@ import org.apache.hadoop.hbase.client.HB
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.After;
import org.junit.AfterClass;
@@ -143,9 +145,13 @@ public class TestExportSnapshot {
@Test
public void testBalanceSplit() throws Exception {
// Create a list of files
- List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
+ List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
for (long i = 0; i <= 20; i++) {
- files.add(new Pair<Path, Long>(new Path("file-" + i), i));
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.HFILE)
+ .setHfile("file-" + i)
+ .build();
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i));
}
// Create 5 groups (total size 210)
@@ -154,18 +160,26 @@ public class TestExportSnapshot {
// group 2: 18, 13, 8, 3 (total size: 42)
// group 3: 17, 12, 7, 4 (total size: 42)
// group 4: 16, 11, 6, 5 (total size: 42)
- List<List<Path>> splits = ExportSnapshot.getBalancedSplits(files, 5);
+ List<List<SnapshotFileInfo>> splits = ExportSnapshot.getBalancedSplits(files, 5);
assertEquals(5, splits.size());
- assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"),
- new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0));
- assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"),
- new Path("file-9"), new Path("file-2")), splits.get(1));
- assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"),
- new Path("file-8"), new Path("file-3")), splits.get(2));
- assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"),
- new Path("file-7"), new Path("file-4")), splits.get(3));
- assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"),
- new Path("file-6"), new Path("file-5")), splits.get(4));
+
+ String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"};
+ verifyBalanceSplit(splits.get(0), split0);
+ String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"};
+ verifyBalanceSplit(splits.get(1), split1);
+ String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"};
+ verifyBalanceSplit(splits.get(2), split2);
+ String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"};
+ verifyBalanceSplit(splits.get(3), split3);
+ String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"};
+ verifyBalanceSplit(splits.get(4), split4);
+ }
+
+ private void verifyBalanceSplit(final List<SnapshotFileInfo> split, final String[] expected) {
+ assertEquals(expected.length, split.size());
+ for (int i = 0; i < expected.length; ++i) {
+ assertEquals(expected[i], split.get(i).getHfile());
+ }
}
/**
@@ -233,6 +247,11 @@ public class TestExportSnapshot {
r1fs.commitStoreFile(TEST_FAMILY, storeFile);
Path tableDir = FSUtils.getTableDir(archiveDir, tableWithRefsName);
+ HTableDescriptor htd = new HTableDescriptor(tableWithRefsName);
+ htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
+ new FSTableDescriptors(fs, rootDir)
+ .createTableDescriptorForTableDirectory(tableDir, htd, false);
+
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
@@ -345,13 +364,14 @@ public class TestExportSnapshot {
new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
- SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
+ SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfile = storeFile.getName();
verifyNonEmptyFile(new Path(exportedArchive,
new Path(FSUtils.getTableDir(new Path("./"), tableName),
- new Path(region, new Path(family, hfile)))));
+ new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
}
public void recoveredEdits (final String region, final String logfile)
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java Wed May 7 21:28:12 2014
@@ -40,7 +40,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
@@ -53,10 +52,8 @@ import org.apache.hadoop.hbase.master.HM
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.log4j.Level;
@@ -153,15 +150,6 @@ public class TestFlushSnapshotFromClient
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before snapshot:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -184,7 +172,7 @@ public class TestFlushSnapshotFromClient
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
}
@@ -202,15 +190,6 @@ public class TestFlushSnapshotFromClient
HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
- // get the name of all the regionservers hosting the snapshotted table
- Set<String> snapshotServers = new HashSet<String>();
- List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
- for (RegionServerThread server : servers) {
- if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
- snapshotServers.add(server.getRegionServer().getServerName().toString());
- }
- }
-
LOG.debug("FS state before snapshot:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
@@ -238,7 +217,7 @@ public class TestFlushSnapshotFromClient
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
+ admin, fs);
}
@Test (timeout=300000)
@@ -399,46 +378,11 @@ public class TestFlushSnapshotFromClient
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
String snapshotName = "flushSnapshotCreateListDestroy";
- // test creating the snapshot
- admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
- logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
-
- // make sure we only have 1 matching snapshot
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
- snapshotName, TABLE_NAME);
-
- // check the directory structure
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
- assertTrue(fs.exists(snapshotDir));
- FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
- Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
- assertTrue(fs.exists(snapshotinfo));
-
- // check the table info
- HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
- rootDir, TABLE_NAME);
- HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
- new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
- assertEquals(desc, snapshotDesc);
-
- // check the region snapshot for all the regions
- List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
- assertTrue(regions.size() > 1);
- for (HRegionInfo info : regions) {
- String regionName = info.getEncodedName();
- Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
- assertEquals(info, snapshotRegionInfo);
- // check to make sure we have the family
- Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
- assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
- fs.exists(familyDir));
-
- // make sure we have some file references
- assertTrue(fs.listStatus(familyDir).length > 0);
- }
+ SnapshotTestingUtils.createSnapshotAndValidate(admin,
+ TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
+ snapshotName, rootDir, fs, true);
}
/**