You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/05/07 23:28:13 UTC
svn commit: r1593139 [4/6] - in /hbase/trunk:
hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/
hbase-protocol/src/main/protobuf/
hbase-server/src/main/java/org/apache/hadoop/hbase/client/
hbase-server/src/main/java/org/apache/ha...
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java Wed May 7 21:28:12 2014
@@ -47,18 +47,21 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.io.FileLink;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HLogLink;
import org.apache.hadoop.hbase.mapreduce.JobUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
@@ -99,7 +102,8 @@ public final class ExportSnapshot extend
// Export Map-Reduce Counters, to keep track of the progress
public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED, FILES_COPIED };
- private static class ExportMapper extends Mapper<Text, NullWritable, NullWritable, NullWritable> {
+ private static class ExportMapper extends Mapper<BytesWritable, NullWritable,
+ NullWritable, NullWritable> {
final static int REPORT_SIZE = 1 * 1024 * 1024;
final static int BUFFER_SIZE = 64 * 1024;
@@ -155,35 +159,35 @@ public final class ExportSnapshot extend
}
@Override
- public void map(Text key, NullWritable value, Context context)
+ public void map(BytesWritable key, NullWritable value, Context context)
throws InterruptedException, IOException {
- Path inputPath = new Path(key.toString());
- Path outputPath = getOutputPath(inputPath);
+ SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(key.copyBytes());
+ Path outputPath = getOutputPath(inputInfo);
- LOG.info("copy file input=" + inputPath + " output=" + outputPath);
- copyFile(context, inputPath, outputPath);
+ copyFile(context, inputInfo, outputPath);
}
/**
* Returns the location where the inputPath will be copied.
- * - hfiles are encoded as hfile links hfile-region-table
- * - logs are encoded as serverName/logName
*/
- private Path getOutputPath(final Path inputPath) throws IOException {
- Path path;
- if (HFileLink.isHFileLink(inputPath) || StoreFileInfo.isReference(inputPath)) {
- String family = inputPath.getParent().getName();
- TableName table =
- HFileLink.getReferencedTableName(inputPath.getName());
- String region = HFileLink.getReferencedRegionName(inputPath.getName());
- String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
- path = new Path(FSUtils.getTableDir(new Path("./"), table),
- new Path(region, new Path(family, hfile)));
- } else if (isHLogLinkPath(inputPath)) {
- String logName = inputPath.getName();
- path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName);
- } else {
- path = inputPath;
+ private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException {
+ Path path = null;
+ switch (inputInfo.getType()) {
+ case HFILE:
+ Path inputPath = new Path(inputInfo.getHfile());
+ String family = inputPath.getParent().getName();
+ TableName table =HFileLink.getReferencedTableName(inputPath.getName());
+ String region = HFileLink.getReferencedRegionName(inputPath.getName());
+ String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
+ path = new Path(FSUtils.getTableDir(new Path("./"), table),
+ new Path(region, new Path(family, hfile)));
+ break;
+ case WAL:
+ Path oldLogsDir = new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME);
+ path = new Path(oldLogsDir, inputInfo.getWalName());
+ break;
+ default:
+ throw new IOException("Invalid File Type: " + inputInfo.getType().toString());
}
return new Path(outputArchive, path);
}
@@ -191,7 +195,7 @@ public final class ExportSnapshot extend
/*
* Used by TestExportSnapshot to simulate a failure
*/
- private void injectTestFailure(final Context context, final Path inputPath)
+ private void injectTestFailure(final Context context, final SnapshotFileInfo inputInfo)
throws IOException {
if (testFailures) {
if (context.getConfiguration().getBoolean(CONF_TEST_RETRY, false)) {
@@ -203,37 +207,38 @@ public final class ExportSnapshot extend
// retry, but at least we reduce the number of test failures due to
// this test exception from the same map task.
if (random.nextFloat() < 0.03) {
- throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath
+ throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputInfo
+ " time=" + System.currentTimeMillis());
}
} else {
context.getCounter(Counter.COPY_FAILED).increment(1);
- throw new IOException("TEST FAILURE: Unable to copy input=" + inputPath);
+ throw new IOException("TEST FAILURE: Unable to copy input=" + inputInfo);
}
}
}
- private void copyFile(final Context context, final Path inputPath, final Path outputPath)
- throws IOException {
- injectTestFailure(context, inputPath);
+ private void copyFile(final Context context, final SnapshotFileInfo inputInfo,
+ final Path outputPath) throws IOException {
+ injectTestFailure(context, inputInfo);
// Get the file information
- FileStatus inputStat = getSourceFileStatus(context, inputPath);
+ FileStatus inputStat = getSourceFileStatus(context, inputInfo);
// Verify if the output file exists and is the same that we want to copy
if (outputFs.exists(outputPath)) {
FileStatus outputStat = outputFs.getFileStatus(outputPath);
if (outputStat != null && sameFile(inputStat, outputStat)) {
- LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
+ LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file.");
return;
}
}
- InputStream in = openSourceFile(context, inputPath);
+ InputStream in = openSourceFile(context, inputInfo);
int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100);
if (Integer.MAX_VALUE != bandwidthMB) {
in = new ThrottledInputStream(new BufferedInputStream(in), bandwidthMB * 1024 * 1024);
}
+
try {
context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
@@ -241,7 +246,7 @@ public final class ExportSnapshot extend
outputFs.mkdirs(outputPath.getParent());
FSDataOutputStream out = outputFs.create(outputPath, true);
try {
- copyData(context, inputPath, in, outputPath, out, inputStat.getLen());
+ copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen());
} finally {
out.close();
}
@@ -275,7 +280,7 @@ public final class ExportSnapshot extend
try {
if (filesMode > 0 && stat.getPermission().toShort() != filesMode) {
outputFs.setPermission(path, new FsPermission(filesMode));
- } else if (!stat.getPermission().equals(refStat.getPermission())) {
+ } else if (refStat != null && !stat.getPermission().equals(refStat.getPermission())) {
outputFs.setPermission(path, refStat.getPermission());
}
} catch (IOException e) {
@@ -283,8 +288,9 @@ public final class ExportSnapshot extend
return false;
}
- String user = stringIsNotEmpty(filesUser) ? filesUser : refStat.getOwner();
- String group = stringIsNotEmpty(filesGroup) ? filesGroup : refStat.getGroup();
+ boolean hasRefStat = (refStat != null);
+ String user = stringIsNotEmpty(filesUser) || !hasRefStat ? filesUser : refStat.getOwner();
+ String group = stringIsNotEmpty(filesGroup) || !hasRefStat ? filesGroup : refStat.getGroup();
if (stringIsNotEmpty(user) || stringIsNotEmpty(group)) {
try {
if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
@@ -367,40 +373,53 @@ public final class ExportSnapshot extend
* Throws an IOException if the communication with the inputFs fail or
* if the file is not found.
*/
- private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException {
+ private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
+ throws IOException {
try {
- if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
- return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
- } else if (isHLogLinkPath(path)) {
- String serverName = path.getParent().getName();
- String logName = path.getName();
- return new HLogLink(inputRoot, serverName, logName).open(inputFs);
+ FileLink link = null;
+ switch (fileInfo.getType()) {
+ case HFILE:
+ Path inputPath = new Path(fileInfo.getHfile());
+ link = new HFileLink(inputRoot, inputArchive, inputPath);
+ break;
+ case WAL:
+ String serverName = fileInfo.getWalServer();
+ String logName = fileInfo.getWalName();
+ link = new HLogLink(inputRoot, serverName, logName);
+ break;
+ default:
+ throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
}
- return inputFs.open(path);
+ return link.open(inputFs);
} catch (IOException e) {
context.getCounter(Counter.MISSING_FILES).increment(1);
- LOG.error("Unable to open source file=" + path, e);
+ LOG.error("Unable to open source file=" + fileInfo.toString(), e);
throw e;
}
}
- private FileStatus getSourceFileStatus(Context context, final Path path) throws IOException {
+ private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
+ throws IOException {
try {
- if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) {
- HFileLink link = new HFileLink(inputRoot, inputArchive, path);
- return link.getFileStatus(inputFs);
- } else if (isHLogLinkPath(path)) {
- String serverName = path.getParent().getName();
- String logName = path.getName();
- return new HLogLink(inputRoot, serverName, logName).getFileStatus(inputFs);
+ FileLink link = null;
+ switch (fileInfo.getType()) {
+ case HFILE:
+ Path inputPath = new Path(fileInfo.getHfile());
+ link = new HFileLink(inputRoot, inputArchive, inputPath);
+ break;
+ case WAL:
+ link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
+ break;
+ default:
+ throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
}
- return inputFs.getFileStatus(path);
+ return link.getFileStatus(inputFs);
} catch (FileNotFoundException e) {
context.getCounter(Counter.MISSING_FILES).increment(1);
- LOG.error("Unable to get the status for source file=" + path, e);
+ LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
throw e;
} catch (IOException e) {
- LOG.error("Unable to get the status for source file=" + path, e);
+ LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
throw e;
}
}
@@ -434,49 +453,54 @@ public final class ExportSnapshot extend
return inChecksum.equals(outChecksum);
}
-
- /**
- * HLog files are encoded as serverName/logName
- * and since all the other files should be in /hbase/table/..path..
- * we can rely on the depth, for now.
- */
- private static boolean isHLogLinkPath(final Path path) {
- return path.depth() == 2;
- }
}
/**
* Extract the list of files (HFiles/HLogs) to copy using Map-Reduce.
* @return list of files referenced by the snapshot (pair of path and size)
*/
- private List<Pair<Path, Long>> getSnapshotFiles(final FileSystem fs, final Path snapshotDir)
- throws IOException {
+ private List<Pair<SnapshotFileInfo, Long>> getSnapshotFiles(final FileSystem fs,
+ final Path snapshotDir) throws IOException {
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- final List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
- final TableName table =
- TableName.valueOf(snapshotDesc.getTable());
+ final List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
+ final TableName table = TableName.valueOf(snapshotDesc.getTable());
final Configuration conf = getConf();
// Get snapshot files
- SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- Path path = HFileLink.createPath(table, region, family, hfile);
- long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
- files.add(new Pair<Path, Long>(path, size));
- }
+ SnapshotReferenceUtil.visitReferencedFiles(conf, fs, snapshotDir, snapshotDesc,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (storeFile.hasReference()) {
+ // copied as part of the manifest
+ } else {
+ String region = regionInfo.getEncodedName();
+ String hfile = storeFile.getName();
+ Path path = HFileLink.createPath(table, region, family, hfile);
+
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.HFILE)
+ .setHfile(path.toString())
+ .build();
- public void recoveredEdits (final String region, final String logfile)
- throws IOException {
- // copied with the snapshot referenecs
+ long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
+ }
}
+ @Override
public void logFile (final String server, final String logfile)
throws IOException {
+ SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
+ .setType(SnapshotFileInfo.Type.WAL)
+ .setWalServer(server)
+ .setWalName(logfile)
+ .build();
+
long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
- files.add(new Pair<Path, Long>(new Path(server, logfile), size));
+ files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
}
});
@@ -491,34 +515,35 @@ public final class ExportSnapshot extend
* and then each group fetch the bigger file available, iterating through groups
* alternating the direction.
*/
- static List<List<Path>> getBalancedSplits(final List<Pair<Path, Long>> files, int ngroups) {
+ static List<List<SnapshotFileInfo>> getBalancedSplits(
+ final List<Pair<SnapshotFileInfo, Long>> files, final int ngroups) {
// Sort files by size, from small to big
- Collections.sort(files, new Comparator<Pair<Path, Long>>() {
- public int compare(Pair<Path, Long> a, Pair<Path, Long> b) {
+ Collections.sort(files, new Comparator<Pair<SnapshotFileInfo, Long>>() {
+ public int compare(Pair<SnapshotFileInfo, Long> a, Pair<SnapshotFileInfo, Long> b) {
long r = a.getSecond() - b.getSecond();
return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
}
});
// create balanced groups
- List<List<Path>> fileGroups = new LinkedList<List<Path>>();
+ List<List<SnapshotFileInfo>> fileGroups = new LinkedList<List<SnapshotFileInfo>>();
long[] sizeGroups = new long[ngroups];
int hi = files.size() - 1;
int lo = 0;
- List<Path> group;
+ List<SnapshotFileInfo> group;
int dir = 1;
int g = 0;
while (hi >= lo) {
if (g == fileGroups.size()) {
- group = new LinkedList<Path>();
+ group = new LinkedList<SnapshotFileInfo>();
fileGroups.add(group);
} else {
group = fileGroups.get(g);
}
- Pair<Path, Long> fileInfo = files.get(hi--);
+ Pair<SnapshotFileInfo, Long> fileInfo = files.get(hi--);
// add the hi one
sizeGroups[g] += fileInfo.getSecond();
@@ -558,25 +583,25 @@ public final class ExportSnapshot extend
* and the number of the files to copy.
*/
private static Path[] createInputFiles(final Configuration conf, final Path inputFolderPath,
- final List<Pair<Path, Long>> snapshotFiles, int mappers)
+ final List<Pair<SnapshotFileInfo, Long>> snapshotFiles, int mappers)
throws IOException, InterruptedException {
FileSystem fs = inputFolderPath.getFileSystem(conf);
LOG.debug("Input folder location: " + inputFolderPath);
- List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);
+ List<List<SnapshotFileInfo>> splits = getBalancedSplits(snapshotFiles, mappers);
Path[] inputFiles = new Path[splits.size()];
- Text key = new Text();
+ BytesWritable key = new BytesWritable();
for (int i = 0; i < inputFiles.length; i++) {
- List<Path> files = splits.get(i);
+ List<SnapshotFileInfo> files = splits.get(i);
inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i));
SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i],
- Text.class, NullWritable.class);
+ BytesWritable.class, NullWritable.class);
LOG.debug("Input split: " + i);
try {
- for (Path file: files) {
- LOG.debug(file.toString());
- key.set(file.toString());
+ for (SnapshotFileInfo file: files) {
+ byte[] pbFileInfo = file.toByteArray();
+ key.set(pbFileInfo, 0, pbFileInfo.length);
writer.append(key, NullWritable.get());
}
} finally {
@@ -591,7 +616,7 @@ public final class ExportSnapshot extend
* Run Map-Reduce Job to perform the files copy.
*/
private void runCopyJob(final Path inputRoot, final Path outputRoot,
- final List<Pair<Path, Long>> snapshotFiles, final boolean verifyChecksum,
+ final List<Pair<SnapshotFileInfo, Long>> snapshotFiles, final boolean verifyChecksum,
final String filesUser, final String filesGroup, final int filesMode,
final int mappers, final int bandwidthMB)
throws IOException, InterruptedException, ClassNotFoundException {
@@ -704,7 +729,7 @@ public final class ExportSnapshot extend
System.err.println("UNEXPECTED: " + cmd);
printUsageAndExit();
}
- } catch (Exception e) {
+ } catch (IOException e) {
printUsageAndExit();
}
}
@@ -761,7 +786,7 @@ public final class ExportSnapshot extend
// Step 0 - Extract snapshot files to copy
LOG.info("Loading Snapshot hfile list");
- final List<Pair<Path, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
+ final List<Pair<SnapshotFileInfo, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
if (mappers == 0 && files.size() > 0) {
mappers = 1 + (files.size() / conf.getInt(CONF_MAP_GROUP, 10));
mappers = Math.min(mappers, files.size());
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java Wed May 7 21:28:12 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.snapshot
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -46,17 +47,18 @@ import org.apache.hadoop.hbase.catalog.C
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.IOUtils;
@@ -115,9 +117,9 @@ public class RestoreSnapshotHelper {
private final ForeignExceptionDispatcher monitor;
private final MonitoredTask status;
+ private final SnapshotManifest snapshotManifest;
private final SnapshotDescription snapshotDesc;
private final TableName snapshotTable;
- private final Path snapshotDir;
private final HTableDescriptor tableDesc;
private final Path rootDir;
@@ -128,8 +130,7 @@ public class RestoreSnapshotHelper {
public RestoreSnapshotHelper(final Configuration conf,
final FileSystem fs,
- final SnapshotDescription snapshotDescription,
- final Path snapshotDir,
+ final SnapshotManifest manifest,
final HTableDescriptor tableDescriptor,
final Path rootDir,
final ForeignExceptionDispatcher monitor,
@@ -137,9 +138,9 @@ public class RestoreSnapshotHelper {
{
this.fs = fs;
this.conf = conf;
- this.snapshotDesc = snapshotDescription;
- this.snapshotTable = TableName.valueOf(snapshotDescription.getTable());
- this.snapshotDir = snapshotDir;
+ this.snapshotManifest = manifest;
+ this.snapshotDesc = manifest.getSnapshotDescription();
+ this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
this.tableDesc = tableDescriptor;
this.rootDir = rootDir;
this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
@@ -153,14 +154,19 @@ public class RestoreSnapshotHelper {
*/
public RestoreMetaChanges restoreHdfsRegions() throws IOException {
LOG.debug("starting restore");
- Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
- if (snapshotRegionNames == null) {
+
+ Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
+ if (regionManifests == null) {
LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
return null;
}
RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
+ // Take a copy of the manifest.keySet() since we are going to modify
+ // this instance, by removing the regions already present in the restore dir.
+ Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
+
// Identify which region are still available and which not.
// NOTE: we rely upon the region name as: "table name, start key, end key"
List<HRegionInfo> tableRegions = getTableRegions();
@@ -168,9 +174,9 @@ public class RestoreSnapshotHelper {
monitor.rethrowException();
for (HRegionInfo regionInfo: tableRegions) {
String regionName = regionInfo.getEncodedName();
- if (snapshotRegionNames.contains(regionName)) {
+ if (regionNames.contains(regionName)) {
LOG.info("region to restore: " + regionName);
- snapshotRegionNames.remove(regionName);
+ regionNames.remove(regionName);
metaChanges.addRegionToRestore(regionInfo);
} else {
LOG.info("region to remove: " + regionName);
@@ -181,7 +187,7 @@ public class RestoreSnapshotHelper {
// Restore regions using the snapshot data
monitor.rethrowException();
status.setStatus("Restoring table regions...");
- restoreHdfsRegions(metaChanges.getRegionsToRestore());
+ restoreHdfsRegions(regionManifests, metaChanges.getRegionsToRestore());
status.setStatus("Finished restoring all table regions.");
// Remove regions from the current table
@@ -192,30 +198,23 @@ public class RestoreSnapshotHelper {
}
// Regions to Add: present in the snapshot but not in the current table
- if (snapshotRegionNames.size() > 0) {
- List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
+ if (regionNames.size() > 0) {
+ List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
monitor.rethrowException();
- for (String regionName: snapshotRegionNames) {
+ for (String regionName: regionNames) {
LOG.info("region to add: " + regionName);
- Path regionDir = new Path(snapshotDir, regionName);
- regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
+ regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
}
// Create new regions cloning from the snapshot
monitor.rethrowException();
status.setStatus("Cloning regions...");
- HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
+ HRegionInfo[] clonedRegions = cloneHdfsRegions(regionManifests, regionsToAdd);
metaChanges.setNewRegions(clonedRegions);
status.setStatus("Finished cloning regions.");
}
- // Restore WALs
- monitor.rethrowException();
- status.setStatus("Restoring WALs to table...");
- restoreWALs();
- status.setStatus("Finished restoring WALs to table.");
-
return metaChanges;
}
@@ -357,19 +356,34 @@ public class RestoreSnapshotHelper {
/**
* Restore specified regions by restoring content to the snapshot state.
*/
- private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+ private void restoreHdfsRegions(final Map<String, SnapshotRegionManifest> regionManifests,
+ final List<HRegionInfo> regions) throws IOException {
if (regions == null || regions.size() == 0) return;
- for (HRegionInfo hri: regions) restoreRegion(hri);
+ for (HRegionInfo hri: regions) {
+ restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
+ }
+ }
+
+ private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
+ final SnapshotRegionManifest manifest) {
+ Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
+ new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
+ for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+ familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
+ new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
+ }
+ return familyMap;
}
/**
* Restore region by removing files not in the snapshot
* and adding the missing ones from the snapshot.
*/
- private void restoreRegion(HRegionInfo regionInfo) throws IOException {
- Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
- Map<String, List<String>> snapshotFiles =
- SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
+ private void restoreRegion(final HRegionInfo regionInfo,
+ final SnapshotRegionManifest regionManifest) throws IOException {
+ Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
+ getRegionHFileReferences(regionManifest);
+
Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
String tableName = tableDesc.getTableName().getNameAsString();
@@ -377,32 +391,34 @@ public class RestoreSnapshotHelper {
for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
byte[] family = Bytes.toBytes(familyDir.getName());
Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
- List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
+ List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
+ snapshotFiles.remove(familyDir.getName());
if (snapshotFamilyFiles != null) {
- List<String> hfilesToAdd = new LinkedList<String>();
- for (String hfileName: snapshotFamilyFiles) {
- if (familyFiles.contains(hfileName)) {
+ List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
+ new ArrayList<SnapshotRegionManifest.StoreFile>();
+ for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
+ if (familyFiles.contains(storeFile.getName())) {
// HFile already present
- familyFiles.remove(hfileName);
+ familyFiles.remove(storeFile.getName());
} else {
// HFile missing
- hfilesToAdd.add(hfileName);
+ hfilesToAdd.add(storeFile);
}
}
// Remove hfiles not present in the snapshot
for (String hfileName: familyFiles) {
Path hfile = new Path(familyDir, hfileName);
- LOG.trace("Removing hfile=" + hfile +
+ LOG.trace("Removing hfile=" + hfileName +
" from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
}
// Restore Missing files
- for (String hfileName: hfilesToAdd) {
- LOG.trace("Adding HFileLink " + hfileName +
+ for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
+ LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, hfileName);
+ restoreStoreFile(familyDir, regionInfo, storeFile);
}
} else {
// Family doesn't exists in the snapshot
@@ -414,15 +430,16 @@ public class RestoreSnapshotHelper {
}
// Add families not present in the table
- for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
+ for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
+ snapshotFiles.entrySet()) {
Path familyDir = new Path(regionDir, familyEntry.getKey());
if (!fs.mkdirs(familyDir)) {
throw new IOException("Unable to create familyDir=" + familyDir);
}
- for (String hfileName: familyEntry.getValue()) {
- LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, hfileName);
+ for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
+ LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
+ restoreStoreFile(familyDir, regionInfo, storeFile);
}
}
}
@@ -448,7 +465,8 @@ public class RestoreSnapshotHelper {
* Clone specified regions. For each region create a new region
* and create a HFileLink for each hfile.
*/
- private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
+ private HRegionInfo[] cloneHdfsRegions(final Map<String, SnapshotRegionManifest> regionManifests,
+ final List<HRegionInfo> regions) throws IOException {
if (regions == null || regions.size() == 0) return null;
final Map<String, HRegionInfo> snapshotRegions =
@@ -476,7 +494,8 @@ public class RestoreSnapshotHelper {
tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
@Override
public void fillRegion(final HRegion region) throws IOException {
- cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
+ HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
+ cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
}
});
@@ -494,21 +513,17 @@ public class RestoreSnapshotHelper {
* @param region {@link HRegion} cloned
* @param snapshotRegionInfo
*/
- private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
- throws IOException {
- final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
+ private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
+ final SnapshotRegionManifest manifest) throws IOException {
final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
final String tableName = tableDesc.getTableName().getNameAsString();
- SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
- new FSVisitor.StoreFileVisitor() {
- @Override
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
- Path familyDir = new Path(regionDir, family);
- restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
- }
- });
+ for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
+ Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
+ for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
+ LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
+ restoreStoreFile(familyDir, snapshotRegionInfo, storeFile);
+ }
+ }
}
/**
@@ -524,11 +539,12 @@ public class RestoreSnapshotHelper {
* @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
*/
private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
- final String hfileName) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfileName = storeFile.getName();
if (HFileLink.isHFileLink(hfileName)) {
HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
} else if (StoreFileInfo.isReference(hfileName)) {
- restoreReferenceFile(familyDir, regionInfo, hfileName);
+ restoreReferenceFile(familyDir, regionInfo, storeFile);
} else {
HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
}
@@ -553,7 +569,9 @@ public class RestoreSnapshotHelper {
* @param hfileName reference file name
*/
private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
- final String hfileName) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ String hfileName = storeFile.getName();
+
// Extract the referred information (hfile name and parent region)
Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(
snapshotTable.getNameAsString(), regionInfo.getEncodedName()), familyDir.getName()),
@@ -577,16 +595,21 @@ public class RestoreSnapshotHelper {
Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
// Create the new reference
- InputStream in;
- if (linkPath != null) {
- in = new HFileLink(conf, linkPath).open(fs);
+ if (storeFile.hasReference()) {
+ Reference reference = Reference.convert(storeFile.getReference());
+ reference.write(fs, outPath);
} else {
- linkPath = new Path(new Path(HRegion.getRegionDir(snapshotDir, regionInfo.getEncodedName()),
- familyDir.getName()), hfileName);
- in = fs.open(linkPath);
+ InputStream in;
+ if (linkPath != null) {
+ in = new HFileLink(conf, linkPath).open(fs);
+ } else {
+ linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
+ regionInfo.getEncodedName()), familyDir.getName()), hfileName);
+ in = fs.open(linkPath);
+ }
+ OutputStream out = fs.create(outPath);
+ IOUtils.copyBytes(in, out, conf);
}
- OutputStream out = fs.create(outPath);
- IOUtils.copyBytes(in, out, conf);
// Add the daughter region to the map
String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
@@ -619,43 +642,6 @@ public class RestoreSnapshotHelper {
}
/**
- * Restore snapshot WALs.
- *
- * Global Snapshot keep a reference to region servers logs present during the snapshot.
- * (/hbase/.snapshot/snapshotName/.logs/hostName/logName)
- *
- * Since each log contains different tables data, logs must be split to
- * extract the table that we are interested in.
- */
- private void restoreWALs() throws IOException {
- final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
- snapshotTable, regionsMap);
- // TODO: use executors to parallelize splitting
- // TODO: once split, we do not need to split again for other restores
- try {
- // Recover.Edits
- SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
- new FSVisitor.RecoveredEditsVisitor() {
- @Override
- public void recoveredEdits (final String region, final String logfile) throws IOException {
- Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
- logSplitter.splitRecoveredEdit(path);
- }
- });
-
- // Region Server Logs
- SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
- @Override
- public void logFile (final String server, final String logfile) throws IOException {
- logSplitter.splitLog(server, logfile);
- }
- });
- } finally {
- logSplitter.close();
- }
- }
-
- /**
* @return the set of the regions contained in the table
*/
private List<HRegionInfo> getTableRegions() throws IOException {
@@ -720,16 +706,14 @@ public class RestoreSnapshotHelper {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
-
- //load table descriptor
- HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
MonitoredTask status = TaskMonitor.get().createStatus(
"Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
- RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, snapshotDesc,
- snapshotDir, htd, restoreDir, monitor, status);
+ RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
+ manifest, manifest.getTableDescriptor(), restoreDir, monitor, status);
helper.restoreHdfsRegions(); // TODO: parallelize.
if (LOG.isDebugEnabled()) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java Wed May 7 21:28:12 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.snapshot
import java.io.IOException;
import java.util.Collections;
-import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,6 +31,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -91,7 +91,7 @@ public class SnapshotDescriptionUtils {
* Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
* which we may need to read in differently.
*/
- public static final int SNAPSHOT_LAYOUT_VERSION = 0;
+ public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
// snapshot directory constants
/**
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java Wed May 7 21:28:12 2014
@@ -31,8 +31,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -43,8 +43,8 @@ import org.apache.hadoop.hbase.HTableDes
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HLogLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
/**
* Tool for dumping snapshot information.
@@ -184,10 +184,10 @@ public final class SnapshotInfo extends
* @param hfile store file name
* @return the store file information
*/
- FileInfo addStoreFile(final String region, final String family, final String hfile)
- throws IOException {
- TableName table = snapshotTable;
- HFileLink link = HFileLink.create(conf, table, region, family, hfile);
+ FileInfo addStoreFile(final HRegionInfo region, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(),
+ family, storeFile.getName());
boolean inArchive = false;
long size = -1;
try {
@@ -207,22 +207,6 @@ public final class SnapshotInfo extends
}
/**
- * Add the specified recovered.edits file to the stats
- * @param region region encoded name
- * @param logfile log file name
- * @return the recovered.edits information
- */
- FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
- Path rootDir = FSUtils.getRootDir(conf);
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
- Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
- long size = fs.getFileStatus(path).getLen();
- logSize += size;
- logsCount++;
- return new FileInfo(true, size);
- }
-
- /**
* Add the specified log file to the stats
* @param server server name
* @param logfile log file name
@@ -245,9 +229,7 @@ public final class SnapshotInfo extends
private FileSystem fs;
private Path rootDir;
- private HTableDescriptor snapshotTableDesc;
- private SnapshotDescription snapshotDesc;
- private Path snapshotDir;
+ private SnapshotManifest snapshotManifest;
@Override
public int run(String[] args) throws IOException, InterruptedException {
@@ -309,14 +291,14 @@ public final class SnapshotInfo extends
* @return false if snapshot is not found
*/
private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
- snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
if (!fs.exists(snapshotDir)) {
LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
return false;
}
- snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+ SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
return true;
}
@@ -324,12 +306,13 @@ public final class SnapshotInfo extends
* Dump the {@link SnapshotDescription}
*/
private void printInfo() {
+ SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
System.out.println("Snapshot Info");
System.out.println("----------------------------------------");
System.out.println(" Name: " + snapshotDesc.getName());
System.out.println(" Type: " + snapshotDesc.getType());
- System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
+ System.out.println(" Table: " + snapshotDesc.getTable());
System.out.println(" Format: " + snapshotDesc.getVersion());
System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
System.out.println();
@@ -341,7 +324,7 @@ public final class SnapshotInfo extends
private void printSchema() {
System.out.println("Table Descriptor");
System.out.println("----------------------------------------");
- System.out.println(snapshotTableDesc.toString());
+ System.out.println(snapshotManifest.getTableDescriptor().toString());
System.out.println();
}
@@ -356,32 +339,26 @@ public final class SnapshotInfo extends
}
// Collect information about hfiles and logs in the snapshot
- final String table = snapshotTableDesc.getTableName().getNameAsString();
- final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
- SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
+ final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
+ final String table = snapshotDesc.getTable();
+ final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
+ SnapshotReferenceUtil.visitReferencedFiles(getConf(), fs,
+ snapshotManifest.getSnapshotDir(), snapshotDesc, new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (storeFile.hasReference()) return;
+ SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
if (showFiles) {
System.out.printf("%8s %s/%s/%s/%s %s%n",
(info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
- table, region, family, hfile,
+ table, regionInfo.getEncodedName(), family, storeFile.getName(),
(info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
}
}
- public void recoveredEdits (final String region, final String logfile)
- throws IOException {
- SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
-
- if (showFiles) {
- System.out.printf("%8s recovered.edits %s on region %s%n",
- StringUtils.humanReadableInt(info.getSize()), logfile, region);
- }
- }
-
+ @Override
public void logFile (final String server, final String logfile)
throws IOException {
SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
@@ -444,17 +421,17 @@ public final class SnapshotInfo extends
FileSystem fs = FileSystem.get(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
- SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
- new SnapshotReferenceUtil.FileVisitor() {
- public void storeFile (final String region, final String family, final String hfile)
- throws IOException {
- stats.addStoreFile(region, family, hfile);
- }
-
- public void recoveredEdits (final String region, final String logfile) throws IOException {
- stats.addRecoveredEdits(region, logfile);
+ SnapshotReferenceUtil.visitReferencedFiles(conf, fs, snapshotDir, snapshot,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (!storeFile.hasReference()) {
+ stats.addStoreFile(regionInfo, family, storeFile);
+ }
}
+ @Override
public void logFile (final String server, final String logfile) throws IOException {
stats.addLogFile(server, logfile);
}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java?rev=1593139&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java Wed May 7 21:28:12 2014
@@ -0,0 +1,468 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.Threads;
+
+/**
+ * Utility class to help read/write the Snapshot Manifest.
+ *
+ * The snapshot format is transparent for the users of this class,
+ * once the snapshot is written, it will never be modified.
+ * On open() the snapshot will be loaded to the current in-memory format.
+ */
+@InterfaceAudience.Private
+public class SnapshotManifest {
+ private static final Log LOG = LogFactory.getLog(SnapshotManifest.class);
+
+ private static final String DATA_MANIFEST_NAME = "data.manifest";
+
+ private List<SnapshotRegionManifest> regionManifests;
+ private SnapshotDescription desc;
+ private HTableDescriptor htd;
+
+ private final ForeignExceptionSnare monitor;
+ private final Configuration conf;
+ private final Path workingDir;
+ private final FileSystem fs;
+
+ private SnapshotManifest(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc,
+ final ForeignExceptionSnare monitor) {
+ this.monitor = monitor;
+ this.desc = desc;
+ this.workingDir = workingDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ /**
+ * Return a SnapshotManifest instance, used for writing a snapshot.
+ *
+ * There are two usage pattern:
+ * - The Master will create a manifest, add the descriptor, offline regions
+ * and consolidate the snapshot by writing all the pending stuff on-disk.
+ * manifest = SnapshotManifest.create(...)
+ * manifest.addRegion(tableDir, hri)
+ * manifest.consolidate()
+ * - The RegionServer will create a single region manifest
+ * manifest = SnapshotManifest.create(...)
+ * manifest.addRegion(region)
+ */
+ public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc,
+ final ForeignExceptionSnare monitor) {
+ return new SnapshotManifest(conf, fs, workingDir, desc, monitor);
+ }
+
+ /**
+ * Return a SnapshotManifest instance with the information already loaded in-memory.
+ * SnapshotManifest manifest = SnapshotManifest.open(...)
+ * HTableDescriptor htd = manifest.getTableDescriptor()
+ * for (SnapshotRegionManifest regionManifest: manifest.getRegionManifests())
+ * hri = regionManifest.getRegionInfo()
+ * for (regionManifest.getFamilyFiles())
+ * ...
+ */
+ public static SnapshotManifest open(final Configuration conf, final FileSystem fs,
+ final Path workingDir, final SnapshotDescription desc) throws IOException {
+ SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null);
+ manifest.load();
+ return manifest;
+ }
+
+
+ /**
+ * Add the table descriptor to the snapshot manifest
+ */
+ public void addTableDescriptor(final HTableDescriptor htd) throws IOException {
+ this.htd = htd;
+ }
+
+ interface RegionVisitor<TRegion, TFamily> {
+ TRegion regionOpen(final HRegionInfo regionInfo) throws IOException;
+ void regionClose(final TRegion region) throws IOException;
+
+ TFamily familyOpen(final TRegion region, final byte[] familyName) throws IOException;
+ void familyClose(final TRegion region, final TFamily family) throws IOException;
+
+ void storeFile(final TRegion region, final TFamily family, final StoreFileInfo storeFile)
+ throws IOException;
+ }
+
+ private RegionVisitor createRegionVisitor(final SnapshotDescription desc) throws IOException {
+ switch (getSnapshotFormat(desc)) {
+ case SnapshotManifestV1.DESCRIPTOR_VERSION:
+ return new SnapshotManifestV1.ManifestBuilder(conf, fs, workingDir);
+ case SnapshotManifestV2.DESCRIPTOR_VERSION:
+ return new SnapshotManifestV2.ManifestBuilder(conf, fs, workingDir);
+ default:
+ throw new CorruptedSnapshotException("Invalid Snapshot version: "+ desc.getVersion(), desc);
+ }
+ }
+
+ /**
+ * Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
+ * This is used by the "online snapshot" when the table is enabled.
+ */
+ public void addRegion(final HRegion region) throws IOException {
+ // 0. Get the ManifestBuilder/RegionVisitor
+ RegionVisitor visitor = createRegionVisitor(desc);
+
+ // 1. dump region meta info into the snapshot directory
+ LOG.debug("Storing '" + region + "' region-info for snapshot.");
+ Object regionData = visitor.regionOpen(region.getRegionInfo());
+ monitor.rethrowException();
+
+ // 2. iterate through all the stores in the region
+ LOG.debug("Creating references for hfiles");
+
+ for (Store store : region.getStores().values()) {
+ // 2.1. build the snapshot reference for the store
+ Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
+ monitor.rethrowException();
+
+ List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+ }
+
+ // 2.2. iterate through all the store's files and create "references".
+ for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
+ StoreFile storeFile = storeFiles.get(i);
+ monitor.rethrowException();
+
+ // create "reference" to this store file.
+ LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath());
+ visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
+ }
+ visitor.familyClose(regionData, familyData);
+ }
+ visitor.regionClose(regionData);
+ }
+
+ /**
+ * Creates a 'manifest' for the specified region, by reading directly from the disk.
+ * This is used by the "offline snapshot" when the table is disabled.
+ */
+ public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ // 0. Get the ManifestBuilder/RegionVisitor
+ RegionVisitor visitor = createRegionVisitor(desc);
+
+ // Open the RegionFS
+ HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
+ tableDir, regionInfo, true);
+ monitor.rethrowException();
+
+ // 1. dump region meta info into the snapshot directory
+ LOG.debug("Storing region-info for snapshot.");
+ Object regionData = visitor.regionOpen(regionInfo);
+ monitor.rethrowException();
+
+ // 2. iterate through all the stores in the region
+ LOG.debug("Creating references for hfiles");
+
+ // This ensures that we have an atomic view of the directory as long as we have < ls limit
+ // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
+ // batches and may miss files being added/deleted. This could be more robust (iteratively
+ // checking to see if we have all the files until we are sure), but the limit is currently 1000
+ // files/batch, far more than the number of store files under a single column family.
+ Collection<String> familyNames = regionFs.getFamilies();
+ if (familyNames != null) {
+ for (String familyName: familyNames) {
+ Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
+ monitor.rethrowException();
+
+ Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
+ if (storeFiles == null) {
+ LOG.debug("No files under family: " + familyName);
+ continue;
+ }
+
+ // 2.1. build the snapshot reference for the store
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+ }
+
+ // 2.2. iterate through all the store's files and create "references".
+ int i = 0;
+ int sz = storeFiles.size();
+ for (StoreFileInfo storeFile: storeFiles) {
+ monitor.rethrowException();
+
+ // create "reference" to this store file.
+ LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
+ visitor.storeFile(regionData, familyData, storeFile);
+ }
+ visitor.familyClose(regionData, familyData);
+ }
+ }
+ visitor.regionClose(regionData);
+ }
+
+ /**
+ * Load the information in the SnapshotManifest. Called by SnapshotManifest.open()
+ *
+ * If the format is v2 and there is no data-manifest, means that we are loading an
+ * in-progress snapshot. Since we support rolling-upgrades, we loook for v1 and v2
+ * regions format.
+ */
+ private void load() throws IOException {
+ switch (getSnapshotFormat(desc)) {
+ case SnapshotManifestV1.DESCRIPTOR_VERSION: {
+ this.htd = FSTableDescriptors.getTableDescriptorFromFs(fs, workingDir);
+ ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ try {
+ this.regionManifests =
+ SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ } finally {
+ tpool.shutdown();
+ }
+ break;
+ }
+ case SnapshotManifestV2.DESCRIPTOR_VERSION: {
+ SnapshotDataManifest dataManifest = readDataManifest();
+ if (dataManifest != null) {
+ htd = HTableDescriptor.convert(dataManifest.getTableSchema());
+ regionManifests = dataManifest.getRegionManifestsList();
+ } else {
+ // Compatibility, load the v1 regions
+ // This happens only when the snapshot is in-progress and the cache wants to refresh.
+ List<SnapshotRegionManifest> v1Regions, v2Regions;
+ ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ try {
+ v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ } finally {
+ tpool.shutdown();
+ }
+ if (v1Regions != null && v2Regions != null) {
+ regionManifests =
+ new ArrayList<SnapshotRegionManifest>(v1Regions.size() + v2Regions.size());
+ regionManifests.addAll(v1Regions);
+ regionManifests.addAll(v2Regions);
+ } else if (v1Regions != null) {
+ regionManifests = v1Regions;
+ } else /* if (v2Regions != null) */ {
+ regionManifests = v2Regions;
+ }
+ }
+ break;
+ }
+ default:
+ throw new CorruptedSnapshotException("Invalid Snapshot version: "+ desc.getVersion(), desc);
+ }
+ }
+
+ /**
+ * Get the current snapshot working dir
+ */
+ public Path getSnapshotDir() {
+ return this.workingDir;
+ }
+
+ /**
+ * Get the SnapshotDescription
+ */
+ public SnapshotDescription getSnapshotDescription() {
+ return this.desc;
+ }
+
+ /**
+ * Get the table descriptor from the Snapshot
+ */
+ public HTableDescriptor getTableDescriptor() {
+ return this.htd;
+ }
+
+ /**
+ * Get all the Region Manifest from the snapshot
+ */
+ public List<SnapshotRegionManifest> getRegionManifests() {
+ return this.regionManifests;
+ }
+
+ /**
+ * Get all the Region Manifest from the snapshot.
+ * This is an helper to get a map with the region encoded name
+ */
+ public Map<String, SnapshotRegionManifest> getRegionManifestsMap() {
+ if (regionManifests == null || regionManifests.size() == 0) return null;
+
+ HashMap<String, SnapshotRegionManifest> regionsMap =
+ new HashMap<String, SnapshotRegionManifest>(regionManifests.size());
+ for (SnapshotRegionManifest manifest: regionManifests) {
+ String regionName = getRegionNameFromManifest(manifest);
+ regionsMap.put(regionName, manifest);
+ }
+ return regionsMap;
+ }
+
+ public void consolidate() throws IOException {
+ if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) {
+ Path rootDir = FSUtils.getRootDir(conf);
+ LOG.info("Using old Snapshot Format");
+ // write a copy of descriptor to the snapshot directory
+ new FSTableDescriptors(fs, rootDir)
+ .createTableDescriptorForTableDirectory(workingDir, htd, false);
+ } else {
+ LOG.debug("Convert to Single Snapshot Manifest");
+ convertToV2SingleManifest();
+ }
+ }
+
+ /*
+ * In case of rolling-upgrade, we try to read all the formats and build
+ * the snapshot with the latest format.
+ */
+ private void convertToV2SingleManifest() throws IOException {
+ // Try to load v1 and v2 regions
+ List<SnapshotRegionManifest> v1Regions, v2Regions;
+ ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
+ try {
+ v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
+ } finally {
+ tpool.shutdown();
+ }
+
+ SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
+ dataManifestBuilder.setTableSchema(htd.convert());
+
+ if (v1Regions != null && v1Regions.size() > 0) {
+ dataManifestBuilder.addAllRegionManifests(v1Regions);
+ }
+ if (v2Regions != null && v2Regions.size() > 0) {
+ dataManifestBuilder.addAllRegionManifests(v2Regions);
+ }
+
+ // Write the v2 Data Manifest.
+ // Once the data-manifest is written, the snapshot can be considered complete.
+ // Currently snapshots are written in a "temporary" directory and later
+ // moved to the "complated" snapshot directory.
+ SnapshotDataManifest dataManifest = dataManifestBuilder.build();
+ writeDataManifest(dataManifest);
+ this.regionManifests = dataManifest.getRegionManifestsList();
+
+ // Remove the region manifests. Everything is now in the data-manifest.
+ // The delete operation is "relaxed", unless we get an exception we keep going.
+ // The extra files in the snapshot directory will not give any problem,
+ // since they have the same content as the data manifest, and even by re-reading
+ // them we will get the same information.
+ if (v1Regions != null && v1Regions.size() > 0) {
+ for (SnapshotRegionManifest regionManifest: v1Regions) {
+ SnapshotManifestV1.deleteRegionManifest(fs, workingDir, regionManifest);
+ }
+ }
+ if (v2Regions != null && v2Regions.size() > 0) {
+ for (SnapshotRegionManifest regionManifest: v2Regions) {
+ SnapshotManifestV2.deleteRegionManifest(fs, workingDir, regionManifest);
+ }
+ }
+ }
+
+ /*
+ * Write the SnapshotDataManifest file
+ */
+ private void writeDataManifest(final SnapshotDataManifest manifest)
+ throws IOException {
+ FSDataOutputStream stream = fs.create(new Path(workingDir, DATA_MANIFEST_NAME));
+ try {
+ manifest.writeTo(stream);
+ } finally {
+ stream.close();
+ }
+ }
+
+ /*
+ * Read the SnapshotDataManifest file
+ */
+ private SnapshotDataManifest readDataManifest() throws IOException {
+ FSDataInputStream in = null;
+ try {
+ in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME));
+ return SnapshotDataManifest.parseFrom(in);
+ } catch (FileNotFoundException e) {
+ return null;
+ } finally {
+ if (in != null) in.close();
+ }
+ }
+
+ private ThreadPoolExecutor createExecutor(final String name) {
+ return createExecutor(conf, name);
+ }
+
+ static ThreadPoolExecutor createExecutor(final Configuration conf, final String name) {
+ int maxThreads = conf.getInt("hbase.snapshot.thread.pool.max", 4);
+ return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+ Threads.getNamedThreadFactory(name));
+ }
+
+ /**
+ * Extract the region encoded name from the region manifest
+ */
+ static String getRegionNameFromManifest(final SnapshotRegionManifest manifest) {
+ byte[] regionName = HRegionInfo.createRegionName(
+ ProtobufUtil.toTableName(manifest.getRegionInfo().getTableName()),
+ manifest.getRegionInfo().getStartKey().toByteArray(),
+ manifest.getRegionInfo().getRegionId(), true);
+ return HRegionInfo.encodeRegionName(regionName);
+ }
+
+ /*
+ * Return the snapshot format
+ */
+ private static int getSnapshotFormat(final SnapshotDescription desc) {
+ return desc.hasVersion() ? desc.getVersion() : SnapshotManifestV1.DESCRIPTOR_VERSION;
+ }
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java?rev=1593139&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java Wed May 7 21:28:12 2014
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import com.google.protobuf.HBaseZeroCopyByteString;
+
+/**
+ * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
+ *
+ * Snapshot v1 layout format
+ * - Each region in the table is represented by a directory with the .hregioninfo file
+ * /snapshotName/regionName/.hregioninfo
+ * - Each file present in the table is represented by an empty file
+ * /snapshotName/regionName/familyName/fileName
+ */
+@InterfaceAudience.Private
+public class SnapshotManifestV1 {
+ private static final Log LOG = LogFactory.getLog(SnapshotManifestV1.class);
+
+ public static final int DESCRIPTOR_VERSION = 0;
+
+ private SnapshotManifestV1() {
+ }
+
+ static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
+ HRegionFileSystem, Path> {
+ private final Configuration conf;
+ private final Path snapshotDir;
+ private final FileSystem fs;
+
+ public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
+ this.snapshotDir = snapshotDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
+ fs, snapshotDir, regionInfo);
+ return snapshotRegionFs;
+ }
+
+ public void regionClose(final HRegionFileSystem region) {
+ }
+
+ public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
+ Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
+ return familyDir;
+ }
+
+ public void familyClose(final HRegionFileSystem region, final Path family) {
+ }
+
+ public void storeFile(final HRegionFileSystem region, final Path familyDir,
+ final StoreFileInfo storeFile) throws IOException {
+ Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
+ boolean success = true;
+ if (storeFile.isReference()) {
+ // write the Reference object to the snapshot
+ storeFile.getReference().write(fs, referenceFile);
+ } else {
+ // create "reference" to this store file. It is intentionally an empty file -- all
+ // necessary information is captured by its fs location and filename. This allows us to
+ // only figure out what needs to be done via a single nn operation (instead of having to
+ // open and read the files as well).
+ success = fs.createNewFile(referenceFile);
+ }
+ if (!success) {
+ throw new IOException("Failed to create reference file:" + referenceFile);
+ }
+ }
+ }
+
+ static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
+ final Executor executor,final FileSystem fs, final Path snapshotDir,
+ final SnapshotDescription desc) throws IOException {
+ FileStatus[] regions = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
+ if (regions == null) {
+ LOG.info("No regions under directory:" + snapshotDir);
+ return null;
+ }
+
+ final ExecutorCompletionService<SnapshotRegionManifest> completionService =
+ new ExecutorCompletionService<SnapshotRegionManifest>(executor);
+ for (final FileStatus region: regions) {
+ completionService.submit(new Callable<SnapshotRegionManifest>() {
+ @Override
+ public SnapshotRegionManifest call() throws IOException {
+ HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
+ return buildManifestFromDisk(conf, fs, snapshotDir, hri);
+ }
+ });
+ }
+
+ ArrayList<SnapshotRegionManifest> regionsManifest =
+ new ArrayList<SnapshotRegionManifest>(regions.length);
+ try {
+ for (int i = 0; i < regions.length; ++i) {
+ regionsManifest.add(completionService.take().get());
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
+ }
+ return regionsManifest;
+ }
+
+ static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
+ final SnapshotRegionManifest manifest) throws IOException {
+ String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
+ fs.delete(new Path(snapshotDir, regionName), true);
+ }
+
+ static SnapshotRegionManifest buildManifestFromDisk (final Configuration conf,
+ final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
+ tableDir, regionInfo, true);
+ SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
+
+ // 1. dump region meta info into the snapshot directory
+ LOG.debug("Storing region-info for snapshot.");
+ manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
+
+ // 2. iterate through all the stores in the region
+ LOG.debug("Creating references for hfiles");
+
+ // This ensures that we have an atomic view of the directory as long as we have < ls limit
+ // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
+ // batches and may miss files being added/deleted. This could be more robust (iteratively
+ // checking to see if we have all the files until we are sure), but the limit is currently 1000
+ // files/batch, far more than the number of store files under a single column family.
+ Collection<String> familyNames = regionFs.getFamilies();
+ if (familyNames != null) {
+ for (String familyName: familyNames) {
+ Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
+ if (storeFiles == null) {
+ LOG.debug("No files under family: " + familyName);
+ continue;
+ }
+
+ // 2.1. build the snapshot reference for the store
+ SnapshotRegionManifest.FamilyFiles.Builder family =
+ SnapshotRegionManifest.FamilyFiles.newBuilder();
+ family.setFamilyName(HBaseZeroCopyByteString.wrap(Bytes.toBytes(familyName)));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+ }
+
+ // 2.2. iterate through all the store's files and create "references".
+ int i = 0;
+ int sz = storeFiles.size();
+ for (StoreFileInfo storeFile: storeFiles) {
+ // create "reference" to this store file.
+ LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
+ SnapshotRegionManifest.StoreFile.Builder sfManifest =
+ SnapshotRegionManifest.StoreFile.newBuilder();
+ sfManifest.setName(storeFile.getPath().getName());
+ family.addStoreFiles(sfManifest.build());
+ }
+ manifest.addFamilyFiles(family.build());
+ }
+ }
+ return manifest.build();
+ }
+}
\ No newline at end of file
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java?rev=1593139&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java Wed May 7 21:28:12 2014
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import com.google.protobuf.HBaseZeroCopyByteString;
+
+/**
+ * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
+ *
+ * Snapshot v2 layout format
+ * - Single Manifest file containing all the information of regions
+ * - In the online-snapshot case each region will write a "region manifest"
+ * /snapshotName/manifest.regionName
+ */
+@InterfaceAudience.Private
+public class SnapshotManifestV2 {
+ private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
+
+ public static final int DESCRIPTOR_VERSION = 2;
+
+ private static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
+
+ static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
+ SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
+ private final Configuration conf;
+ private final Path snapshotDir;
+ private final FileSystem fs;
+
+ public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
+ this.snapshotDir = snapshotDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
+ SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
+ manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
+ return manifest;
+ }
+
+ public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
+ SnapshotRegionManifest manifest = region.build();
+ FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
+ try {
+ manifest.writeTo(stream);
+ } finally {
+ stream.close();
+ }
+ }
+
+ public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
+ final SnapshotRegionManifest.Builder region, final byte[] familyName) {
+ SnapshotRegionManifest.FamilyFiles.Builder family =
+ SnapshotRegionManifest.FamilyFiles.newBuilder();
+ family.setFamilyName(HBaseZeroCopyByteString.wrap(familyName));
+ return family;
+ }
+
+ public void familyClose(final SnapshotRegionManifest.Builder region,
+ final SnapshotRegionManifest.FamilyFiles.Builder family) {
+ region.addFamilyFiles(family.build());
+ }
+
+ public void storeFile(final SnapshotRegionManifest.Builder region,
+ final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile) {
+ SnapshotRegionManifest.StoreFile.Builder sfManifest =
+ SnapshotRegionManifest.StoreFile.newBuilder();
+ sfManifest.setName(storeFile.getPath().getName());
+ if (storeFile.isReference()) {
+ sfManifest.setReference(storeFile.getReference().convert());
+ }
+ sfManifest.setFileSize(storeFile.getFileStatus().getLen());
+ family.addStoreFiles(sfManifest.build());
+ }
+ }
+
+ static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
+ final Executor executor,final FileSystem fs, final Path snapshotDir,
+ final SnapshotDescription desc) throws IOException {
+ FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
+ }
+ });
+
+ if (manifestFiles == null || manifestFiles.length == 0) return null;
+
+ final ExecutorCompletionService<SnapshotRegionManifest> completionService =
+ new ExecutorCompletionService<SnapshotRegionManifest>(executor);
+ for (final FileStatus st: manifestFiles) {
+ completionService.submit(new Callable<SnapshotRegionManifest>() {
+ @Override
+ public SnapshotRegionManifest call() throws IOException {
+ FSDataInputStream stream = fs.open(st.getPath());
+ try {
+ return SnapshotRegionManifest.parseFrom(stream);
+ } finally {
+ stream.close();
+ }
+ }
+ });
+ }
+
+ ArrayList<SnapshotRegionManifest> regionsManifest =
+ new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
+ try {
+ for (int i = 0; i < manifestFiles.length; ++i) {
+ regionsManifest.add(completionService.take().get());
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
+ }
+ return regionsManifest;
+ }
+
+ static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
+ final SnapshotRegionManifest manifest) throws IOException {
+ fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
+ }
+
+ private static Path getRegionManifestPath(final Path snapshotDir,
+ final SnapshotRegionManifest manifest) {
+ String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
+ return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
+ }
+}