You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/11/30 06:56:21 UTC
[3/8] hbase git commit: HBASE-16904 Snapshot related changes for FS
redo work
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
deleted file mode 100644
index 8c24b1e..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ /dev/null
@@ -1,823 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.RegionStorage;
-import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
-import org.apache.hadoop.hbase.fs.legacy.io.HFileLink;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ModifyRegionUtils;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.io.IOUtils;
-
-/**
- * Helper to Restore/Clone a Snapshot
- *
- * <p>The helper assumes that a table is already created, and by calling restore()
- * the content present in the snapshot will be restored as the new content of the table.
- *
- * <p>Clone from Snapshot: If the target table is empty, the restore operation
- * is just a "clone operation", where the only operations are:
- * <ul>
- * <li>for each region in the snapshot create a new region
- * (note that the region will have a different name, since the encoding contains the table name)
- * <li>for each file in the region create a new HFileLink to point to the original file.
- * <li>restore the logs, if any
- * </ul>
- *
- * <p>Restore from Snapshot:
- * <ul>
- * <li>for each region in the table verify which are available in the snapshot and which are not
- * <ul>
- * <li>if the region is not present in the snapshot, remove it.
- * <li>if the region is present in the snapshot
- * <ul>
- * <li>for each file in the table region verify which are available in the snapshot
- * <ul>
- * <li>if the hfile is not present in the snapshot, remove it
- * <li>if the hfile is present, keep it (nothing to do)
- * </ul>
- * <li>for each file in the snapshot region but not in the table
- * <ul>
- * <li>create a new HFileLink that point to the original file
- * </ul>
- * </ul>
- * </ul>
- * <li>for each region in the snapshot not present in the current table state
- * <ul>
- * <li>create a new region and for each file in the region create a new HFileLink
- * (This is the same as the clone operation)
- * </ul>
- * <li>restore the logs, if any
- * </ul>
- *
- * TODO update for MasterStorage / RegionStorage
- */
-@InterfaceAudience.Private
-public class RestoreSnapshotHelper {
- private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
-
- private final Map<byte[], byte[]> regionsMap =
- new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
-
- private final Map<String, Pair<String, String> > parentsMap =
- new HashMap<String, Pair<String, String> >();
-
- private final ForeignExceptionDispatcher monitor;
- private final MonitoredTask status;
-
- private final SnapshotManifest snapshotManifest;
- private final SnapshotDescription snapshotDesc;
- private final TableName snapshotTable;
-
- private final HTableDescriptor tableDesc;
- private final Path tableDir;
-
- private final Configuration conf;
- private final FileSystem fs;
- private final boolean createBackRefs;
-
- public RestoreSnapshotHelper(final Configuration conf, final SnapshotManifest manifest,
- final HTableDescriptor tableDescriptor, final ForeignExceptionDispatcher monitor,
- final MonitoredTask status) throws IOException {
- this(conf, manifest, tableDescriptor, monitor, status, true);
- }
-
- public RestoreSnapshotHelper(final Configuration conf, final SnapshotManifest manifest,
- final HTableDescriptor tableDescriptor, final ForeignExceptionDispatcher monitor,
- final MonitoredTask status, final boolean createBackRefs) throws IOException {
- this.fs = FSUtils.getCurrentFileSystem(conf);
- this.conf = conf;
- this.snapshotManifest = manifest;
- this.snapshotDesc = manifest.getSnapshotDescription();
- this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
- this.tableDesc = tableDescriptor;
- this.tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableDesc.getTableName());
- this.monitor = monitor;
- this.status = status;
- this.createBackRefs = createBackRefs;
- }
-
- /**
- * Restore the on-disk table to a specified snapshot state.
- * @return the set of regions touched by the restore operation
- */
- public RestoreMetaChanges restoreStorageRegions() throws IOException {
- ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
- try {
- return restoreHdfsRegions(exec);
- } finally {
- exec.shutdown();
- }
- }
-
- private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
- LOG.info("starting restore table regions using snapshot=" + snapshotDesc);
-
- Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
- if (regionManifests == null) {
- LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
- return null;
- }
-
- RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap);
-
- // Take a copy of the manifest.keySet() since we are going to modify
- // this instance, by removing the regions already present in the restore dir.
- Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
-
- HRegionInfo mobRegion = MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor()
- .getTableName());
- // Identify which region are still available and which not.
- // NOTE: we rely upon the region name as: "table name, start key, end key"
- List<HRegionInfo> tableRegions = getTableRegions();
- if (tableRegions != null) {
- monitor.rethrowException();
- for (HRegionInfo regionInfo: tableRegions) {
- String regionName = regionInfo.getEncodedName();
- if (regionNames.contains(regionName)) {
- LOG.info("region to restore: " + regionName);
- regionNames.remove(regionName);
- metaChanges.addRegionToRestore(regionInfo);
- } else {
- LOG.info("region to remove: " + regionName);
- metaChanges.addRegionToRemove(regionInfo);
- }
- }
-
- // Restore regions using the snapshot data
- monitor.rethrowException();
- status.setStatus("Restoring table regions...");
- if (regionNames.contains(mobRegion.getEncodedName())) {
- // restore the mob region in case
- List<HRegionInfo> mobRegions = new ArrayList<HRegionInfo>(1);
- mobRegions.add(mobRegion);
- restoreHdfsMobRegions(exec, regionManifests, mobRegions);
- regionNames.remove(mobRegion.getEncodedName());
- }
- restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
- status.setStatus("Finished restoring all table regions.");
-
- // Remove regions from the current table
- monitor.rethrowException();
- status.setStatus("Starting to delete excess regions from table");
- removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
- status.setStatus("Finished deleting excess regions from table.");
- }
-
- // Regions to Add: present in the snapshot but not in the current table
- if (regionNames.size() > 0) {
- List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
-
- monitor.rethrowException();
- // add the mob region
- if (regionNames.contains(mobRegion.getEncodedName())) {
- cloneHdfsMobRegion(regionManifests, mobRegion);
- regionNames.remove(mobRegion.getEncodedName());
- }
- for (String regionName: regionNames) {
- LOG.info("region to add: " + regionName);
- regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
- }
-
- // Create new regions cloning from the snapshot
- monitor.rethrowException();
- status.setStatus("Cloning regions...");
- HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
- metaChanges.setNewRegions(clonedRegions);
- status.setStatus("Finished cloning regions.");
- }
-
- LOG.info("finishing restore table regions using snapshot=" + snapshotDesc);
-
- return metaChanges;
- }
-
- /**
- * Describe the set of operations needed to update hbase:meta after restore.
- */
- public static class RestoreMetaChanges {
- private final Map<String, Pair<String, String> > parentsMap;
- private final HTableDescriptor htd;
-
- private List<HRegionInfo> regionsToRestore = null;
- private List<HRegionInfo> regionsToRemove = null;
- private List<HRegionInfo> regionsToAdd = null;
-
- public RestoreMetaChanges(HTableDescriptor htd, Map<String, Pair<String, String> > parentsMap) {
- this.parentsMap = parentsMap;
- this.htd = htd;
- }
-
- public HTableDescriptor getTableDescriptor() {
- return htd;
- }
-
- /**
- * Returns the map of parent-children_pair.
- * @return the map
- */
- public Map<String, Pair<String, String>> getParentToChildrenPairMap() {
- return this.parentsMap;
- }
-
- /**
- * @return true if there're new regions
- */
- public boolean hasRegionsToAdd() {
- return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
- }
-
- /**
- * Returns the list of new regions added during the on-disk restore.
- * The caller is responsible to add the regions to META.
- * e.g MetaTableAccessor.addRegionsToMeta(...)
- * @return the list of regions to add to META
- */
- public List<HRegionInfo> getRegionsToAdd() {
- return this.regionsToAdd;
- }
-
- /**
- * @return true if there're regions to restore
- */
- public boolean hasRegionsToRestore() {
- return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
- }
-
- /**
- * Returns the list of 'restored regions' during the on-disk restore.
- * The caller is responsible to add the regions to hbase:meta if not present.
- * @return the list of regions restored
- */
- public List<HRegionInfo> getRegionsToRestore() {
- return this.regionsToRestore;
- }
-
- /**
- * @return true if there're regions to remove
- */
- public boolean hasRegionsToRemove() {
- return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
- }
-
- /**
- * Returns the list of regions removed during the on-disk restore.
- * The caller is responsible to remove the regions from META.
- * e.g. MetaTableAccessor.deleteRegions(...)
- * @return the list of regions to remove from META
- */
- public List<HRegionInfo> getRegionsToRemove() {
- return this.regionsToRemove;
- }
-
- void setNewRegions(final HRegionInfo[] hris) {
- if (hris != null) {
- regionsToAdd = Arrays.asList(hris);
- } else {
- regionsToAdd = null;
- }
- }
-
- void addRegionToRemove(final HRegionInfo hri) {
- if (regionsToRemove == null) {
- regionsToRemove = new LinkedList<HRegionInfo>();
- }
- regionsToRemove.add(hri);
- }
-
- void addRegionToRestore(final HRegionInfo hri) {
- if (regionsToRestore == null) {
- regionsToRestore = new LinkedList<HRegionInfo>();
- }
- regionsToRestore.add(hri);
- }
-
- public void updateMetaParentRegions(Connection connection,
- final List<HRegionInfo> regionInfos) throws IOException {
- if (regionInfos == null || parentsMap.isEmpty()) return;
-
- // Extract region names and offlined regions
- Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
- List<HRegionInfo> parentRegions = new LinkedList<>();
- for (HRegionInfo regionInfo: regionInfos) {
- if (regionInfo.isSplitParent()) {
- parentRegions.add(regionInfo);
- } else {
- regionsByName.put(regionInfo.getEncodedName(), regionInfo);
- }
- }
-
- // Update Offline parents
- for (HRegionInfo regionInfo: parentRegions) {
- Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
- if (daughters == null) {
- // The snapshot contains an unreferenced region.
- // It will be removed by the CatalogJanitor.
- LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
- continue;
- }
-
- // One side of the split is already compacted
- if (daughters.getSecond() == null) {
- daughters.setSecond(daughters.getFirst());
- }
-
- LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
- MetaTableAccessor.addRegionToMeta(connection, regionInfo,
- regionsByName.get(daughters.getFirst()),
- regionsByName.get(daughters.getSecond()));
- }
- }
- }
-
- /**
- * Remove specified regions from the file-system, using the archiver.
- */
- private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<HRegionInfo> regions)
- throws IOException {
- if (regions == null || regions.size() == 0) return;
- ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
- @Override
- public void editRegion(final HRegionInfo hri) throws IOException {
- HFileArchiver.archiveRegion(conf, fs, hri);
- }
- });
- }
-
- /**
- * Restore specified regions by restoring content to the snapshot state.
- */
- private void restoreHdfsRegions(final ThreadPoolExecutor exec,
- final Map<String, SnapshotRegionManifest> regionManifests,
- final List<HRegionInfo> regions) throws IOException {
- if (regions == null || regions.size() == 0) return;
- ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
- @Override
- public void editRegion(final HRegionInfo hri) throws IOException {
- restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
- }
- });
- }
-
- /**
- * Restore specified mob regions by restoring content to the snapshot state.
- */
- private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
- final Map<String, SnapshotRegionManifest> regionManifests,
- final List<HRegionInfo> regions) throws IOException {
- if (regions == null || regions.size() == 0) return;
- ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
- @Override
- public void editRegion(final HRegionInfo hri) throws IOException {
- restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
- }
- });
- }
-
- private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
- final SnapshotRegionManifest manifest) {
- Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
- new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
- for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
- familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
- new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
- }
- return familyMap;
- }
-
- /**
- * Restore region by removing files not in the snapshot
- * and adding the missing ones from the snapshot.
- */
- private void restoreRegion(final HRegionInfo regionInfo,
- final SnapshotRegionManifest regionManifest) throws IOException {
- restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
- }
-
- /**
- * Restore mob region by removing files not in the snapshot
- * and adding the missing ones from the snapshot.
- */
- private void restoreMobRegion(final HRegionInfo regionInfo,
- final SnapshotRegionManifest regionManifest) throws IOException {
- if (regionManifest == null) {
- return;
- }
- restoreRegion(regionInfo, regionManifest,
- MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
- }
-
- /**
- * Restore region by removing files not in the snapshot
- * and adding the missing ones from the snapshot.
- */
- private void restoreRegion(final HRegionInfo regionInfo,
- final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
- Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
- getRegionHFileReferences(regionManifest);
-
- String tableName = tableDesc.getTableName().getNameAsString();
-
- // Restore families present in the table
- for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
- byte[] family = Bytes.toBytes(familyDir.getName());
- Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
- List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
- snapshotFiles.remove(familyDir.getName());
- if (snapshotFamilyFiles != null) {
- List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
- new ArrayList<SnapshotRegionManifest.StoreFile>();
- for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
- if (familyFiles.contains(storeFile.getName())) {
- // HFile already present
- familyFiles.remove(storeFile.getName());
- } else {
- // HFile missing
- hfilesToAdd.add(storeFile);
- }
- }
-
- // Remove hfiles not present in the snapshot
- for (String hfileName: familyFiles) {
- Path hfile = new Path(familyDir, hfileName);
- LOG.trace("Removing hfile=" + hfileName +
- " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
- HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
- }
-
- // Restore Missing files
- for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
- LOG.debug("Adding HFileLink " + storeFile.getName() +
- " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
- }
- } else {
- // Family doesn't exists in the snapshot
- LOG.trace("Removing family=" + Bytes.toString(family) +
- " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
- HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
- fs.delete(familyDir, true);
- }
- }
-
- // Add families not present in the table
- for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
- snapshotFiles.entrySet()) {
- Path familyDir = new Path(regionDir, familyEntry.getKey());
- if (!fs.mkdirs(familyDir)) {
- throw new IOException("Unable to create familyDir=" + familyDir);
- }
-
- for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
- LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
- }
- }
- }
-
- /**
- * @return The set of files in the specified family directory.
- */
- private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
- FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
- if (hfiles == null) return Collections.emptySet();
-
- Set<String> familyFiles = new HashSet<String>(hfiles.length);
- for (int i = 0; i < hfiles.length; ++i) {
- String hfileName = hfiles[i].getPath().getName();
- familyFiles.add(hfileName);
- }
-
- return familyFiles;
- }
-
- /**
- * Clone specified regions. For each region create a new region
- * and create a HFileLink for each hfile.
- */
- private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
- final Map<String, SnapshotRegionManifest> regionManifests,
- final List<HRegionInfo> regions) throws IOException {
- if (regions == null || regions.size() == 0) return null;
-
- final Map<String, HRegionInfo> snapshotRegions =
- new HashMap<String, HRegionInfo>(regions.size());
-
- // clone region info (change embedded tableName with the new one)
- HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
- for (int i = 0; i < clonedRegionsInfo.length; ++i) {
- // clone the region info from the snapshot region info
- HRegionInfo snapshotRegionInfo = regions.get(i);
- clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
-
- // add the region name mapping between snapshot and cloned
- String snapshotRegionName = snapshotRegionInfo.getEncodedName();
- String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
- regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
- LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
-
- // Add mapping between cloned region name and snapshot region info
- snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
- }
-
- // create the regions on disk
- ModifyRegionUtils.createRegions(exec, conf,
- tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
- @Override
- public void fillRegion(final HRegion region) throws IOException {
- HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
- cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
- }
- });
-
- return clonedRegionsInfo;
- }
-
- /**
- * Clone the mob region. For the region create a new region
- * and create a HFileLink for each hfile.
- */
- private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
- final HRegionInfo region) throws IOException {
- // clone region info (change embedded tableName with the new one)
- Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
- cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
- }
-
- /**
- * Clone region directory content from the snapshot info.
- *
- * Each region is encoded with the table name, so the cloned region will have
- * a different region name.
- *
- * Instead of copying the hfiles a HFileLink is created.
- *
- * @param regionDir {@link Path} cloned dir
- * @param snapshotRegionInfo
- */
- private void cloneRegion(final Path regionDir, final HRegionInfo snapshotRegionInfo,
- final SnapshotRegionManifest manifest) throws IOException {
- final String tableName = tableDesc.getTableName().getNameAsString();
- for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
- Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
- for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
- LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
- restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
- }
- }
- }
-
- /**
- * Clone region directory content from the snapshot info.
- *
- * Each region is encoded with the table name, so the cloned region will have
- * a different region name.
- *
- * Instead of copying the hfiles a HFileLink is created.
- *
- * @param region {@link HRegion} cloned
- * @param snapshotRegionInfo
- */
- private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
- final SnapshotRegionManifest manifest) throws IOException {
- cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo,
- manifest);
- }
-
- /**
- * Create a new {@link HFileLink} to reference the store file.
- * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
- * <ul>
- * <li>hfile: abc -> table=region-abc
- * <li>reference: abc.1234 -> table=region-abc.1234
- * <li>hfilelink: table=region-hfile -> table=region-hfile
- * </ul>
- * @param familyDir destination directory for the store file
- * @param regionInfo destination region info for the table
- * @param createBackRef - Whether back reference should be created. Defaults to true.
- * @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
- */
- private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
- final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
- throws IOException {
- String hfileName = storeFile.getName();
- if (HFileLink.isHFileLink(hfileName)) {
- HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
- } else if (StoreFileInfo.isReference(hfileName)) {
- restoreReferenceFile(familyDir, regionInfo, storeFile);
- } else {
- HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
- }
- }
-
- /**
- * Create a new {@link Reference} as copy of the source one.
- * <p><blockquote><pre>
- * The source table looks like:
- * 1234/abc (original file)
- * 5678/abc.1234 (reference file)
- *
- * After the clone operation looks like:
- * wxyz/table=1234-abc
- * stuv/table=1234-abc.wxyz
- *
- * NOTE that the region name in the clone changes (md5 of regioninfo)
- * and the reference should reflect that change.
- * </pre></blockquote>
- * @param familyDir destination directory for the store file
- * @param regionInfo destination region info for the table
- * @param storeFile reference file name
- */
- private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
- String hfileName = storeFile.getName();
-
- // Extract the referred information (hfile name and parent region)
- Path refPath =
- StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable
- .getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo
- .getEncodedName()), familyDir.getName()), hfileName));
- String snapshotRegionName = refPath.getParent().getParent().getName();
- String fileName = refPath.getName();
-
- // The new reference should have the cloned region name as parent, if it is a clone.
- String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
- if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
-
- // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
- Path linkPath = null;
- String refLink = fileName;
- if (!HFileLink.isHFileLink(fileName)) {
- refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
- linkPath = new Path(familyDir,
- HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
- }
-
- Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
-
- // Create the new reference
- if (storeFile.hasReference()) {
- Reference reference = Reference.convert(storeFile.getReference());
- reference.write(fs, outPath);
- } else {
- InputStream in;
- if (linkPath != null) {
- in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
- } else {
- linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
- regionInfo.getEncodedName()), familyDir.getName()), hfileName);
- in = fs.open(linkPath);
- }
- OutputStream out = fs.create(outPath);
- IOUtils.copyBytes(in, out, conf);
- }
-
- // Add the daughter region to the map
- String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
- LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
- synchronized (parentsMap) {
- Pair<String, String> daughters = parentsMap.get(clonedRegionName);
- if (daughters == null) {
- daughters = new Pair<String, String>(regionName, null);
- parentsMap.put(clonedRegionName, daughters);
- } else if (!regionName.equals(daughters.getFirst())) {
- daughters.setSecond(regionName);
- }
- }
- }
-
- /**
- * Create a new {@link HRegionInfo} from the snapshot region info.
- * Keep the same startKey, endKey, regionId and split information but change
- * the table name.
- *
- * @param snapshotRegionInfo Info for region to clone.
- * @return the new HRegion instance
- */
- public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
- return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo);
- }
-
- public static HRegionInfo cloneRegionInfo(TableName tableName, HRegionInfo snapshotRegionInfo) {
- HRegionInfo regionInfo = new HRegionInfo(tableName,
- snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
- snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
- regionInfo.setOffline(snapshotRegionInfo.isOffline());
- return regionInfo;
- }
-
- /**
- * @return the set of the regions contained in the table
- */
- private List<HRegionInfo> getTableRegions() throws IOException {
- LOG.debug("get table regions: " + tableDir);
- FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
- if (regionDirs == null) return null;
-
- List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
- for (FileStatus regionDir: regionDirs) {
- final RegionStorage rs = RegionStorage.open(conf, new LegacyPathIdentifier(regionDir.getPath()), false);
- regions.add(rs.getRegionInfo());
- }
- LOG.debug("found " + regions.size() + " regions for table=" +
- tableDesc.getTableName().getNameAsString());
- return regions;
- }
-
- /**
- * Copy the snapshot files for a snapshot scanner, discards meta changes.
- * @param conf
- * @param fs
- * @param rootDir
- * @param restoreDir
- * @param snapshotName
- * @throws IOException
- */
- public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs,
- Path rootDir, Path restoreDir, String snapshotName) throws IOException {
- // ensure that restore dir is not under root dir
- if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
- throw new IllegalArgumentException("Filesystems for restore directory and HBase root " +
- "directory should be the same");
- }
- if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
- throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
- "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
- }
-
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
- SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
-
- MonitoredTask status = TaskMonitor.get().createStatus(
- "Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
- ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
-
- // we send createBackRefs=false so that restored hfiles do not create back reference links
- // in the base hbase root dir.
- RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf,
- manifest, manifest.getTableDescriptor(), monitor, status, false);
- RestoreMetaChanges metaChanges = helper.restoreStorageRegions(); // TODO: parallelize.
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Restored table dir:" + restoreDir);
- FSUtils.logFileSystemState(fs, restoreDir, LOG);
- }
- return metaChanges;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index 2fd619d..ce12ba9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -17,79 +17,21 @@
*/
package org.apache.hadoop.hbase.snapshot;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collections;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifestV2;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
/**
* Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
- * <p>
- * Snapshots are laid out on disk like this:
- *
- * <pre>
- * /hbase/.snapshots
- * /.tmp <---- working directory
- * /[snapshot name] <----- completed snapshot
- * </pre>
- *
- * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
- * signified by '...' on the same directory depth).
- *
- * <pre>
- * /hbase/.snapshots/completed
- * .snapshotinfo <--- Description of the snapshot
- * .tableinfo <--- Copy of the tableinfo
- * /.logs
- * /[server_name]
- * /... [log files]
- * ...
- * /[region name] <---- All the region's information
- * .regioninfo <---- Copy of the HRegionInfo
- * /[column family name]
- * /[hfile name] <--- name of the hfile in the real region
- * ...
- * ...
- * ...
- * </pre>
- *
- * Utility methods in this class are useful for getting the correct locations for different parts of
- * the snapshot, as well as moving completed snapshots into place (see
- * {@link #completeSnapshot}, and writing the
- * {@link SnapshotDescription} to the working snapshot directory.
*/
@InterfaceAudience.Private
public final class SnapshotDescriptionUtils {
- /**
- * Filter that only accepts completed snapshot directories
- */
- public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter {
-
- /**
- * @param fs
- */
- public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
- super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME));
- }
- }
-
private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
/**
* Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
@@ -97,17 +39,6 @@ public final class SnapshotDescriptionUtils {
*/
public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
- // snapshot directory constants
- /**
- * The file contains the snapshot basic information and it is under the directory of a snapshot.
- */
- public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
-
- /** Temporary directory under the snapshot directory to store in-progress snapshots */
- public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
-
- /** This tag will be created in in-progess snapshots */
- public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
// snapshot operation values
/** Default value if no start time is specified */
public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
@@ -157,86 +88,6 @@ public final class SnapshotDescriptionUtils {
}
/**
- * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
- * ${hbase.rootdir}/.snapshot
- * @param rootDir hbase root directory
- * @return the base directory in which all snapshots are kept
- */
- public static Path getSnapshotRootDir(final Path rootDir) {
- return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
- }
-
- /**
- * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
- * directory and all the data files for a snapshot are kept under this directory.
- * @param snapshot snapshot being taken
- * @param rootDir hbase root directory
- * @return the final directory for the completed snapshot
- */
- public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
- return getCompletedSnapshotDir(snapshot.getName(), rootDir);
- }
-
- /**
- * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
- * directory and all the data files for a snapshot are kept under this directory.
- * @param snapshotName name of the snapshot being taken
- * @param rootDir hbase root directory
- * @return the final directory for the completed snapshot
- */
- public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
- return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
- }
-
- /**
- * Get the general working directory for snapshots - where they are built, where they are
- * temporarily copied on export, etc.
- * @param rootDir root directory of the HBase installation
- * @return Path to the snapshot tmp directory, relative to the passed root directory
- */
- public static Path getWorkingSnapshotDir(final Path rootDir) {
- return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
- }
-
- /**
- * Get the directory to build a snapshot, before it is finalized
- * @param snapshot snapshot that will be built
- * @param rootDir root directory of the hbase installation
- * @return {@link Path} where one can build a snapshot
- */
- public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
- return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
- }
-
- /**
- * Get the directory to build a snapshot, before it is finalized
- * @param snapshotName name of the snapshot
- * @param rootDir root directory of the hbase installation
- * @return {@link Path} where one can build a snapshot
- */
- public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
- return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
- }
-
- /**
- * Get the directory to store the snapshot instance
- * @param snapshotsDir hbase-global directory for storing all snapshots
- * @param snapshotName name of the snapshot to take
- * @return the final directory for the completed snapshot
- */
- private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
- return new Path(snapshotsDir, snapshotName);
- }
-
- /**
- * @param rootDir hbase root directory
- * @return the directory for all completed snapshots;
- */
- public static final Path getSnapshotsDir(Path rootDir) {
- return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
- }
-
- /**
* Convert the passed snapshot description into a 'full' snapshot description based on default
* parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
* supplied to their default values.
@@ -267,94 +118,6 @@ public final class SnapshotDescriptionUtils {
}
/**
- * Write the snapshot description into the working directory of a snapshot
- * @param snapshot description of the snapshot being taken
- * @param workingDir working directory of the snapshot
- * @param fs {@link FileSystem} on which the snapshot should be taken
- * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
- * failure
- */
- public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
- throws IOException {
- FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
- HConstants.DATA_FILE_UMASK_KEY);
- Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
- try {
- FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
- try {
- snapshot.writeTo(out);
- } finally {
- out.close();
- }
- } catch (IOException e) {
- // if we get an exception, try to remove the snapshot info
- if (!fs.delete(snapshotInfo, false)) {
- String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
- LOG.error(msg);
- throw new IOException(msg);
- }
- }
- }
-
- /**
- * Create in-progress tag under .tmp of in-progress snapshot
- * */
- public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException {
- FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
- HConstants.DATA_FILE_UMASK_KEY);
- Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
- FSUtils.create(fs, snapshot_in_progress, perms, true);
- }
-
- /**
- * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
- * @param fs filesystem where the snapshot was taken
- * @param snapshotDir directory where the snapshot was stored
- * @return the stored snapshot description
- * @throws CorruptedSnapshotException if the
- * snapshot cannot be read
- */
- public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
- throws CorruptedSnapshotException {
- Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
- try {
- FSDataInputStream in = null;
- try {
- in = fs.open(snapshotInfo);
- SnapshotDescription desc = SnapshotDescription.parseFrom(in);
- return desc;
- } finally {
- if (in != null) in.close();
- }
- } catch (IOException e) {
- throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
- }
- }
-
- /**
- * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
- * as 'complete'.
- * @param snapshot description of the snapshot being tabken
- * @param rootdir root directory of the hbase installation
- * @param workingDir directory where the in progress snapshot was built
- * @param fs {@link FileSystem} where the snapshot was built
- * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the
- * snapshot could not be moved
- * @throws IOException the filesystem could not be reached
- */
- public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
- FileSystem fs) throws SnapshotCreationException, IOException {
- Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
- LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
- + finishedDir);
- if (!fs.rename(workingDir, finishedDir)) {
- throw new SnapshotCreationException(
- "Failed to move working directory(" + workingDir + ") to completed directory("
- + finishedDir + ").", ProtobufUtil.createSnapshotDesc(snapshot));
- }
- }
-
- /**
* Check if the user is this table snapshot's owner
* @param snapshot the table snapshot description
* @param user the user
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index 6f7fd8f..b7fb50d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -36,14 +36,20 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -53,7 +59,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.fs.legacy.io.HFileLink;
import org.apache.hadoop.hbase.io.WALLink;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -135,24 +140,21 @@ public final class SnapshotInfo extends Configured implements Tool {
private final HBaseProtos.SnapshotDescription snapshot;
private final TableName snapshotTable;
- private final Configuration conf;
- private final FileSystem fs;
+ private final MasterStorage<? extends StorageIdentifier> masterStorage;
- SnapshotStats(final Configuration conf, final FileSystem fs,
+ SnapshotStats(final MasterStorage<? extends StorageIdentifier> masterStorage,
final SnapshotDescription snapshot)
{
this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
this.snapshotTable = TableName.valueOf(snapshot.getTable());
- this.conf = conf;
- this.fs = fs;
+ this.masterStorage = masterStorage;
}
- SnapshotStats(final Configuration conf, final FileSystem fs,
+ SnapshotStats(final MasterStorage<? extends StorageIdentifier> masterStorage,
final HBaseProtos.SnapshotDescription snapshot) {
this.snapshot = snapshot;
this.snapshotTable = TableName.valueOf(snapshot.getTable());
- this.conf = conf;
- this.fs = fs;
+ this.masterStorage = masterStorage;
}
@@ -225,7 +227,7 @@ public final class SnapshotInfo extends Configured implements Tool {
* with other snapshots and tables
*
* This is only calculated when
- * {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)}
+ * {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)}
* is called with a non-null Map
*/
public long getNonSharedArchivedStoreFilesSize() {
@@ -265,7 +267,7 @@ public final class SnapshotInfo extends Configured implements Tool {
Path parentDir = filePath.getParent();
Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
try {
- if (FSUtils.listStatus(fs, backRefDir) == null) {
+ if (FSUtils.listStatus(masterStorage.getFileSystem(), backRefDir) == null) {
return false;
}
} catch (IOException e) {
@@ -287,6 +289,8 @@ public final class SnapshotInfo extends Configured implements Tool {
FileInfo addStoreFile(final HRegionInfo region, final String family,
final SnapshotRegionManifest.StoreFile storeFile,
final Map<Path, Integer> filesMap) throws IOException {
+ Configuration conf = masterStorage.getConfiguration();
+ FileSystem fs = masterStorage.getFileSystem();
HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
family, storeFile.getName());
boolean isCorrupted = false;
@@ -328,10 +332,10 @@ public final class SnapshotInfo extends Configured implements Tool {
* @return the log information
*/
FileInfo addLogFile(final String server, final String logfile) throws IOException {
- WALLink logLink = new WALLink(conf, server, logfile);
+ WALLink logLink = new WALLink(masterStorage.getConfiguration(), server, logfile);
long size = -1;
try {
- size = logLink.getFileStatus(fs).getLen();
+ size = logLink.getFileStatus(masterStorage.getFileSystem()).getLen();
logSize.addAndGet(size);
logsCount.incrementAndGet();
} catch (FileNotFoundException e) {
@@ -342,10 +346,10 @@ public final class SnapshotInfo extends Configured implements Tool {
}
private boolean printSizeInBytes = false;
- private FileSystem fs;
- private Path rootDir;
+ private MasterStorage<? extends StorageIdentifier> masterStorage;
- private SnapshotManifest snapshotManifest;
+ private HBaseProtos.SnapshotDescription snapshotDesc;
+ private HTableDescriptor snapshotTable;
@Override
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
@@ -391,11 +395,14 @@ public final class SnapshotInfo extends Configured implements Tool {
}
}
+ // instantiate MasterStorage
+ masterStorage = MasterStorage.open(conf, false);
+
// List Available Snapshots
if (listSnapshots) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
- for (SnapshotDescription desc: getSnapshotList(conf)) {
+ for (SnapshotDescription desc: getSnapshotList(masterStorage)) {
System.out.printf("%-20s | %20s | %s%n",
desc.getName(),
df.format(new Date(desc.getCreationTime())),
@@ -410,9 +417,8 @@ public final class SnapshotInfo extends Configured implements Tool {
return 1;
}
- rootDir = FSUtils.getRootDir(conf);
- fs = FileSystem.get(rootDir.toUri(), conf);
- LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
+ LOG.debug("fs=" + masterStorage.getFileSystem().getUri().toString() + " root=" +
+ ((LegacyPathIdentifier)masterStorage.getRootContainer()).path);
// Load snapshot information
if (!loadSnapshotInfo(snapshotName)) {
@@ -433,15 +439,8 @@ public final class SnapshotInfo extends Configured implements Tool {
* @return false if snapshot is not found
*/
private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
- if (!fs.exists(snapshotDir)) {
- LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
- return false;
- }
-
- HBaseProtos.SnapshotDescription snapshotDesc =
- SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
- snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
+ snapshotDesc = masterStorage.getSnapshot(snapshotName);
+ snapshotTable = masterStorage.getTableDescriptorForSnapshot(snapshotDesc);
return true;
}
@@ -449,7 +448,6 @@ public final class SnapshotInfo extends Configured implements Tool {
* Dump the {@link SnapshotDescription}
*/
private void printInfo() {
- HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
System.out.println("Snapshot Info");
System.out.println("----------------------------------------");
@@ -468,7 +466,7 @@ public final class SnapshotInfo extends Configured implements Tool {
private void printSchema() {
System.out.println("Table Descriptor");
System.out.println("----------------------------------------");
- System.out.println(snapshotManifest.getTableDescriptor().toString());
+ System.out.println(snapshotTable.toString());
System.out.println();
}
@@ -483,30 +481,30 @@ public final class SnapshotInfo extends Configured implements Tool {
}
// Collect information about hfiles and logs in the snapshot
- final HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
final String table = snapshotDesc.getTable();
SnapshotDescription desc = new SnapshotDescription(snapshotDesc.getName(),
snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
- final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc);
- SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
- "SnapshotInfo",
- new SnapshotReferenceUtil.SnapshotVisitor() {
- @Override
- public void storeFile(final HRegionInfo regionInfo, final String family,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
- if (storeFile.hasReference()) return;
-
- SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null);
- if (showFiles) {
- String state = info.getStateToString();
- System.out.printf("%8s %s/%s/%s/%s %s%n",
- (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
- table, regionInfo.getEncodedName(), family, storeFile.getName(),
- state == null ? "" : "(" + state + ")");
+ final SnapshotStats stats = new SnapshotStats(masterStorage, desc);
+ // TODO: add concurrent API to MasterStorage
+ masterStorage.visitSnapshotStoreFiles(snapshotDesc, StorageContext.DATA,
+ new MasterStorage.SnapshotStoreFileVisitor() {
+ @Override
+ public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot,
+ StorageContext ctx, HRegionInfo hri, String familyName,
+ SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (storeFile.hasReference()) return;
+
+ SnapshotStats.FileInfo info = stats.addStoreFile(hri, familyName, storeFile, null);
+ if (showFiles) {
+ String state = info.getStateToString();
+ System.out.printf("%8s %s/%s/%s/%s %s%n",
+ (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
+ table, hri.getEncodedName(), familyName, storeFile.getName(),
+ state == null ? "" : "(" + state + ")");
+ }
}
- }
- });
+ });
// Dump the stats
System.out.println();
@@ -569,7 +567,8 @@ public final class SnapshotInfo extends Configured implements Tool {
HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
snapshot);
- return getSnapshotStats(conf, snapshotDesc, null);
+ MasterStorage<? extends StorageIdentifier> masterStorage = MasterStorage.open(conf, false);
+ return getSnapshotStats(masterStorage.getConfiguration(), snapshotDesc, null);
}
/**
@@ -579,53 +578,43 @@ public final class SnapshotInfo extends Configured implements Tool {
* @param filesMap {@link Map} store files map for all snapshots, it may be null
* @return the snapshot stats
*/
- public static SnapshotStats getSnapshotStats(final Configuration conf,
- final HBaseProtos.SnapshotDescription snapshotDesc,
+ public static SnapshotStats getSnapshotStats(
+ final Configuration conf, final HBaseProtos.SnapshotDescription snapshotDesc,
final Map<Path, Integer> filesMap) throws IOException {
- Path rootDir = FSUtils.getRootDir(conf);
- FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
- SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
- final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc);
- SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
- "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() {
+ MasterStorage<? extends StorageIdentifier> masterStorage = MasterStorage.open(conf, false);
+ final SnapshotStats stats = new SnapshotStats(masterStorage, snapshotDesc);
+ masterStorage.visitSnapshotStoreFiles(snapshotDesc, StorageContext.DATA,
+ new MasterStorage.SnapshotStoreFileVisitor() {
@Override
- public void storeFile(final HRegionInfo regionInfo, final String family,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot,
+ StorageContext ctx, HRegionInfo hri, String familyName,
+ SnapshotRegionManifest.StoreFile storeFile) throws IOException {
if (!storeFile.hasReference()) {
- stats.addStoreFile(regionInfo, family, storeFile, filesMap);
+ stats.addStoreFile(hri, familyName, storeFile, filesMap);
}
- }});
+ }
+ });
return stats;
}
/**
* Returns the list of available snapshots in the specified location
- * @param conf the {@link Configuration} to use
+ * @param masterStorage the {@link MasterStorage} to use
* @return the list of snapshots
*/
- public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
- throws IOException {
- Path rootDir = FSUtils.getRootDir(conf);
- FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
- FileStatus[] snapshots = fs.listStatus(snapshotDir,
- new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
- List<SnapshotDescription> snapshotLists =
- new ArrayList<SnapshotDescription>(snapshots.length);
- for (FileStatus snapshotDirStat: snapshots) {
- HBaseProtos.SnapshotDescription snapshotDesc =
- SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath());
- snapshotLists.add(new SnapshotDescription(snapshotDesc.getName(),
- snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
- snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion()));
+ public static List<SnapshotDescription> getSnapshotList(
+ final MasterStorage<? extends StorageIdentifier> masterStorage) throws IOException {
+ List<HBaseProtos.SnapshotDescription> descriptions = masterStorage.getSnapshots();
+ List<SnapshotDescription> snapshotLists = new ArrayList(descriptions.size());
+ for (HBaseProtos.SnapshotDescription desc: masterStorage.getSnapshots()) {
+ snapshotLists.add(ProtobufUtil.createSnapshotDesc(desc));
}
return snapshotLists;
}
/**
* Gets the store files map for snapshot
- * @param conf the {@link Configuration} to use
+ * @param masterStorage the {@link MasterStorage} to use
* @param snapshot {@link SnapshotDescription} to get stats from
* @param exec the {@link ExecutorService} to use
* @param filesMap {@link Map} the map to put the mapping entries
@@ -634,33 +623,32 @@ public final class SnapshotInfo extends Configured implements Tool {
* @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared
* @return the snapshot stats
*/
- private static void getSnapshotFilesMap(final Configuration conf,
+ private static void getSnapshotFilesMap(
+ final MasterStorage<? extends StorageIdentifier> masterStorage,
final SnapshotDescription snapshot, final ExecutorService exec,
final ConcurrentHashMap<Path, Integer> filesMap,
final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize,
final AtomicLong uniqueHFilesMobSize) throws IOException {
HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
snapshot);
- Path rootDir = FSUtils.getRootDir(conf);
- final FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
-
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
- SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
- SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec,
- new SnapshotReferenceUtil.SnapshotVisitor() {
- @Override public void storeFile(final HRegionInfo regionInfo, final String family,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ masterStorage.visitSnapshotStoreFiles(snapshotDesc, StorageContext.DATA,
+ new MasterStorage.SnapshotStoreFileVisitor() {
+ @Override
+ public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot,
+ StorageContext ctx, HRegionInfo hri, String familyName,
+ SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ Configuration conf = masterStorage.getConfiguration();
+ FileSystem fs = masterStorage.getFileSystem();
if (!storeFile.hasReference()) {
- HFileLink link = HFileLink
- .build(conf, TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(),
- family, storeFile.getName());
+ HFileLink link = HFileLink.build(conf, TableName.valueOf(snapshot.getTable()),
+ hri.getEncodedName(), familyName, storeFile.getName());
long size;
Integer count;
Path p;
AtomicLong al;
int c = 0;
- if (fs.exists(link.getArchivePath())) {
+ if (masterStorage.getFileSystem().exists(link.getArchivePath())) {
p = link.getArchivePath();
al = uniqueHFilesArchiveSize;
size = fs.getFileStatus(p).getLen();
@@ -685,7 +673,7 @@ public final class SnapshotInfo extends Configured implements Tool {
filesMap.put(p, ++c);
}
}
- });
+ });
}
/**
@@ -699,7 +687,8 @@ public final class SnapshotInfo extends Configured implements Tool {
public static Map<Path, Integer> getSnapshotsFilesMap(final Configuration conf,
AtomicLong uniqueHFilesArchiveSize, AtomicLong uniqueHFilesSize,
AtomicLong uniqueHFilesMobSize) throws IOException {
- List<SnapshotDescription> snapshotList = getSnapshotList(conf);
+ MasterStorage<? extends StorageIdentifier> masterStorage = MasterStorage.open(conf, false);
+ List<SnapshotDescription> snapshotList = getSnapshotList(masterStorage);
if (snapshotList.size() == 0) {
@@ -712,7 +701,7 @@ public final class SnapshotInfo extends Configured implements Tool {
try {
for (final SnapshotDescription snapshot : snapshotList) {
- getSnapshotFilesMap(conf, snapshot, exec, fileMap, uniqueHFilesArchiveSize,
+ getSnapshotFilesMap(masterStorage, snapshot, exec, fileMap, uniqueHFilesArchiveSize,
uniqueHFilesSize, uniqueHFilesMobSize);
}
} finally {