You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/09/22 16:34:35 UTC
[47/50] [abbrv] hbase git commit: HBASE-14439 break everything HRegion
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
deleted file mode 100644
index 508b4a7..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ /dev/null
@@ -1,1108 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
-
-/**
- * View to an on-disk Region.
- * Provides the set of methods necessary to interact with the on-disk region data.
- */
-@InterfaceAudience.Private
-public class HRegionFileSystem {
- private static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
-
- /** Name of the region info file that resides just under the region directory. */
- public final static String REGION_INFO_FILE = ".regioninfo";
-
- /** Temporary subdirectory of the region directory used for merges. */
- public static final String REGION_MERGES_DIR = ".merges";
-
- /** Temporary subdirectory of the region directory used for splits. */
- public static final String REGION_SPLITS_DIR = ".splits";
-
- /** Temporary subdirectory of the region directory used for compaction output. */
- private static final String REGION_TEMP_DIR = ".tmp";
-
- private final HRegionInfo regionInfo;
- //regionInfo for interacting with FS (getting encodedName, etc)
- private final HRegionInfo regionInfoForFs;
- private final Configuration conf;
- private final Path tableDir;
- private final FileSystem fs;
-
- /**
- * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
- * client level.
- */
- private final int hdfsClientRetriesNumber;
- private final int baseSleepBeforeRetries;
- private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
- private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
-
- /**
- * Create a view to the on-disk region
- * @param conf the {@link Configuration} to use
- * @param fs {@link FileSystem} that contains the region
- * @param tableDir {@link Path} to where the table is being stored
- * @param regionInfo {@link HRegionInfo} for region
- */
- HRegionFileSystem(final Configuration conf, final FileSystem fs, final Path tableDir,
- final HRegionInfo regionInfo) {
- this.fs = fs;
- this.conf = conf;
- this.tableDir = tableDir;
- this.regionInfo = regionInfo;
- this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(regionInfo);
- this.hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
- DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
- this.baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
- DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
- }
-
- /** @return the underlying {@link FileSystem} */
- public FileSystem getFileSystem() {
- return this.fs;
- }
-
- /** @return the {@link HRegionInfo} that describe this on-disk region view */
- public HRegionInfo getRegionInfo() {
- return this.regionInfo;
- }
-
- public HRegionInfo getRegionInfoForFS() {
- return this.regionInfoForFs;
- }
-
- /** @return {@link Path} to the region's root directory. */
- public Path getTableDir() {
- return this.tableDir;
- }
-
- /** @return {@link Path} to the region directory. */
- public Path getRegionDir() {
- return new Path(this.tableDir, this.regionInfoForFs.getEncodedName());
- }
-
- // ===========================================================================
- // Temp Helpers
- // ===========================================================================
- /** @return {@link Path} to the region's temp directory, used for file creations */
- Path getTempDir() {
- return new Path(getRegionDir(), REGION_TEMP_DIR);
- }
-
- /**
- * Clean up any temp detritus that may have been left around from previous operation attempts.
- */
- void cleanupTempDir() throws IOException {
- deleteDir(getTempDir());
- }
-
- // ===========================================================================
- // Store/StoreFile Helpers
- // ===========================================================================
- /**
- * Returns the directory path of the specified family
- * @param familyName Column Family Name
- * @return {@link Path} to the directory of the specified family
- */
- public Path getStoreDir(final String familyName) {
- return new Path(this.getRegionDir(), familyName);
- }
-
- /**
- * Create the store directory for the specified family name
- * @param familyName Column Family Name
- * @return {@link Path} to the directory of the specified family
- * @throws IOException if the directory creation fails.
- */
- Path createStoreDir(final String familyName) throws IOException {
- Path storeDir = getStoreDir(familyName);
- if(!fs.exists(storeDir) && !createDir(storeDir))
- throw new IOException("Failed creating "+storeDir);
- return storeDir;
- }
-
- /**
- * Returns the store files available for the family.
- * This methods performs the filtering based on the valid store files.
- * @param familyName Column Family Name
- * @return a set of {@link StoreFileInfo} for the specified family.
- */
- public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
- return getStoreFiles(Bytes.toString(familyName));
- }
-
- public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
- return getStoreFiles(familyName, true);
- }
-
- /**
- * Returns the store files available for the family.
- * This methods performs the filtering based on the valid store files.
- * @param familyName Column Family Name
- * @return a set of {@link StoreFileInfo} for the specified family.
- */
- public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
- throws IOException {
- Path familyDir = getStoreDir(familyName);
- FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
- if (files == null) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("No StoreFiles for: " + familyDir);
- }
- return null;
- }
-
- ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
- for (FileStatus status: files) {
- if (validate && !StoreFileInfo.isValid(status)) {
- LOG.warn("Invalid StoreFile: " + status.getPath());
- continue;
- }
- StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
- regionInfoForFs, familyName, status.getPath());
- storeFiles.add(info);
-
- }
- return storeFiles;
- }
-
- /**
- * Return Qualified Path of the specified family/file
- *
- * @param familyName Column Family Name
- * @param fileName File Name
- * @return The qualified Path for the specified family/file
- */
- Path getStoreFilePath(final String familyName, final String fileName) {
- Path familyDir = getStoreDir(familyName);
- return new Path(familyDir, fileName).makeQualified(this.fs);
- }
-
- /**
- * Return the store file information of the specified family/file.
- *
- * @param familyName Column Family Name
- * @param fileName File Name
- * @return The {@link StoreFileInfo} for the specified family/file
- */
- StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
- throws IOException {
- Path familyDir = getStoreDir(familyName);
- return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
- regionInfoForFs, familyName, new Path(familyDir, fileName));
- }
-
- /**
- * Returns true if the specified family has reference files
- * @param familyName Column Family Name
- * @return true if family contains reference files
- * @throws IOException
- */
- public boolean hasReferences(final String familyName) throws IOException {
- FileStatus[] files = FSUtils.listStatus(fs, getStoreDir(familyName));
- if (files != null) {
- for(FileStatus stat: files) {
- if(stat.isDirectory()) {
- continue;
- }
- if(StoreFileInfo.isReference(stat.getPath())) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Check whether region has Reference file
- * @param htd table desciptor of the region
- * @return true if region has reference file
- * @throws IOException
- */
- public boolean hasReferences(final HTableDescriptor htd) throws IOException {
- for (HColumnDescriptor family : htd.getFamilies()) {
- if (hasReferences(family.getNameAsString())) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * @return the set of families present on disk
- * @throws IOException
- */
- public Collection<String> getFamilies() throws IOException {
- FileStatus[] fds = FSUtils.listStatus(fs, getRegionDir(), new FSUtils.FamilyDirFilter(fs));
- if (fds == null) return null;
-
- ArrayList<String> families = new ArrayList<String>(fds.length);
- for (FileStatus status: fds) {
- families.add(status.getPath().getName());
- }
-
- return families;
- }
-
- /**
- * Remove the region family from disk, archiving the store files.
- * @param familyName Column Family Name
- * @throws IOException if an error occours during the archiving
- */
- public void deleteFamily(final String familyName) throws IOException {
- // archive family store files
- HFileArchiver.archiveFamily(fs, conf, regionInfoForFs, tableDir, Bytes.toBytes(familyName));
-
- // delete the family folder
- Path familyDir = getStoreDir(familyName);
- if(fs.exists(familyDir) && !deleteDir(familyDir))
- throw new IOException("Could not delete family " + familyName
- + " from FileSystem for region " + regionInfoForFs.getRegionNameAsString() + "("
- + regionInfoForFs.getEncodedName() + ")");
- }
-
- /**
- * Generate a unique file name, used by createTempName() and commitStoreFile()
- * @param suffix extra information to append to the generated name
- * @return Unique file name
- */
- private static String generateUniqueName(final String suffix) {
- String name = UUID.randomUUID().toString().replaceAll("-", "");
- if (suffix != null) name += suffix;
- return name;
- }
-
- /**
- * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
- * to get a safer file creation.
- * <code>
- * Path file = fs.createTempName();
- * ...StoreFile.Writer(file)...
- * fs.commitStoreFile("family", file);
- * </code>
- *
- * @return Unique {@link Path} of the temporary file
- */
- public Path createTempName() {
- return createTempName(null);
- }
-
- /**
- * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
- * to get a safer file creation.
- * <code>
- * Path file = fs.createTempName();
- * ...StoreFile.Writer(file)...
- * fs.commitStoreFile("family", file);
- * </code>
- *
- * @param suffix extra information to append to the generated name
- * @return Unique {@link Path} of the temporary file
- */
- public Path createTempName(final String suffix) {
- return new Path(getTempDir(), generateUniqueName(suffix));
- }
-
- /**
- * Move the file from a build/temp location to the main family store directory.
- * @param familyName Family that will gain the file
- * @param buildPath {@link Path} to the file to commit.
- * @return The new {@link Path} of the committed file
- * @throws IOException
- */
- public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
- return commitStoreFile(familyName, buildPath, -1, false);
- }
-
- /**
- * Move the file from a build/temp location to the main family store directory.
- * @param familyName Family that will gain the file
- * @param buildPath {@link Path} to the file to commit.
- * @param seqNum Sequence Number to append to the file name (less then 0 if no sequence number)
- * @param generateNewName False if you want to keep the buildPath name
- * @return The new {@link Path} of the committed file
- * @throws IOException
- */
- private Path commitStoreFile(final String familyName, final Path buildPath,
- final long seqNum, final boolean generateNewName) throws IOException {
- Path storeDir = getStoreDir(familyName);
- if(!fs.exists(storeDir) && !createDir(storeDir))
- throw new IOException("Failed creating " + storeDir);
-
- String name = buildPath.getName();
- if (generateNewName) {
- name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
- }
- Path dstPath = new Path(storeDir, name);
- if (!fs.exists(buildPath)) {
- throw new FileNotFoundException(buildPath.toString());
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Committing store file " + buildPath + " as " + dstPath);
- }
- // buildPath exists, therefore not doing an exists() check.
- if (!rename(buildPath, dstPath)) {
- throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
- }
- return dstPath;
- }
-
-
- /**
- * Moves multiple store files to the relative region's family store directory.
- * @param storeFiles list of store files divided by family
- * @throws IOException
- */
- void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
- for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
- String familyName = Bytes.toString(es.getKey());
- for (StoreFile sf: es.getValue()) {
- commitStoreFile(familyName, sf.getPath());
- }
- }
- }
-
- /**
- * Archives the specified store file from the specified family.
- * @param familyName Family that contains the store files
- * @param filePath {@link Path} to the store file to remove
- * @throws IOException if the archiving fails
- */
- public void removeStoreFile(final String familyName, final Path filePath)
- throws IOException {
- HFileArchiver.archiveStoreFile(this.conf, this.fs, this.regionInfoForFs,
- this.tableDir, Bytes.toBytes(familyName), filePath);
- }
-
- /**
- * Closes and archives the specified store files from the specified family.
- * @param familyName Family that contains the store files
- * @param storeFiles set of store files to remove
- * @throws IOException if the archiving fails
- */
- public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
- throws IOException {
- HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.regionInfoForFs,
- this.tableDir, Bytes.toBytes(familyName), storeFiles);
- }
-
- /**
- * Bulk load: Add a specified store file to the specified family.
- * If the source file is on the same different file-system is moved from the
- * source location to the destination location, otherwise is copied over.
- *
- * @param familyName Family that will gain the file
- * @param srcPath {@link Path} to the file to import
- * @param seqNum Bulk Load sequence number
- * @return The destination {@link Path} of the bulk loaded file
- * @throws IOException
- */
- Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
- throws IOException {
- // Copy the file if it's on another filesystem
- FileSystem srcFs = srcPath.getFileSystem(conf);
- FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
-
- // We can't compare FileSystem instances as equals() includes UGI instance
- // as part of the comparison and won't work when doing SecureBulkLoad
- // TODO deal with viewFS
- if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) {
- LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
- "the destination store. Copying file over to destination filesystem.");
- Path tmpPath = createTempName();
- FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf);
- LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
- srcPath = tmpPath;
- }
-
- return commitStoreFile(familyName, srcPath, seqNum, true);
- }
-
- // ===========================================================================
- // Splits Helpers
- // ===========================================================================
- /** @return {@link Path} to the temp directory used during split operations */
- Path getSplitsDir() {
- return new Path(getRegionDir(), REGION_SPLITS_DIR);
- }
-
- Path getSplitsDir(final HRegionInfo hri) {
- return new Path(getSplitsDir(), hri.getEncodedName());
- }
-
- /**
- * Clean up any split detritus that may have been left around from previous split attempts.
- */
- void cleanupSplitsDir() throws IOException {
- deleteDir(getSplitsDir());
- }
-
- /**
- * Clean up any split detritus that may have been left around from previous
- * split attempts.
- * Call this method on initial region deploy.
- * @throws IOException
- */
- void cleanupAnySplitDetritus() throws IOException {
- Path splitdir = this.getSplitsDir();
- if (!fs.exists(splitdir)) return;
- // Look at the splitdir. It could have the encoded names of the daughter
- // regions we tried to make. See if the daughter regions actually got made
- // out under the tabledir. If here under splitdir still, then the split did
- // not complete. Try and do cleanup. This code WILL NOT catch the case
- // where we successfully created daughter a but regionserver crashed during
- // the creation of region b. In this case, there'll be an orphan daughter
- // dir in the filesystem. TOOD: Fix.
- FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
- if (daughters != null) {
- for (FileStatus daughter: daughters) {
- Path daughterDir = new Path(getTableDir(), daughter.getPath().getName());
- if (fs.exists(daughterDir) && !deleteDir(daughterDir)) {
- throw new IOException("Failed delete of " + daughterDir);
- }
- }
- }
- cleanupSplitsDir();
- LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
- }
-
- /**
- * Remove daughter region
- * @param regionInfo daughter {@link HRegionInfo}
- * @throws IOException
- */
- void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
- Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
- if (this.fs.exists(regionDir) && !deleteDir(regionDir)) {
- throw new IOException("Failed delete of " + regionDir);
- }
- }
-
- /**
- * Commit a daughter region, moving it from the split temporary directory
- * to the proper location in the filesystem.
- *
- * @param regionInfo daughter {@link org.apache.hadoop.hbase.HRegionInfo}
- * @throws IOException
- */
- Path commitDaughterRegion(final HRegionInfo regionInfo)
- throws IOException {
- Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
- Path daughterTmpDir = this.getSplitsDir(regionInfo);
-
- if (fs.exists(daughterTmpDir)) {
-
- // Write HRI to a file in case we need to recover hbase:meta
- Path regionInfoFile = new Path(daughterTmpDir, REGION_INFO_FILE);
- byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
- writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
-
- // Move the daughter temp dir to the table dir
- if (!rename(daughterTmpDir, regionDir)) {
- throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
- }
- }
-
- return regionDir;
- }
-
- /**
- * Create the region splits directory.
- */
- void createSplitsDir() throws IOException {
- Path splitdir = getSplitsDir();
- if (fs.exists(splitdir)) {
- LOG.info("The " + splitdir + " directory exists. Hence deleting it to recreate it");
- if (!deleteDir(splitdir)) {
- throw new IOException("Failed deletion of " + splitdir
- + " before creating them again.");
- }
- }
- // splitDir doesn't exists now. No need to do an exists() call for it.
- if (!createDir(splitdir)) {
- throw new IOException("Failed create of " + splitdir);
- }
- }
-
- /**
- * Write out a split reference. Package local so it doesnt leak out of
- * regionserver.
- * @param hri {@link HRegionInfo} of the destination
- * @param familyName Column Family Name
- * @param f File to split.
- * @param splitRow Split Row
- * @param top True if we are referring to the top half of the hfile.
- * @param splitPolicy
- * @return Path to created reference.
- * @throws IOException
- */
- Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
- final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
- throws IOException {
- if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
- // Check whether the split row lies in the range of the store file
- // If it is outside the range, return directly.
- try {
- if (top) {
- //check if larger than last key.
- KeyValue splitKey = KeyValueUtil.createFirstOnRow(splitRow);
- Cell lastKey = f.getLastKey();
- // If lastKey is null means storefile is empty.
- if (lastKey == null) {
- return null;
- }
- if (f.getComparator().compare(splitKey, lastKey) > 0) {
- return null;
- }
- } else {
- //check if smaller than first key
- KeyValue splitKey = KeyValueUtil.createLastOnRow(splitRow);
- Cell firstKey = f.getFirstKey();
- // If firstKey is null means storefile is empty.
- if (firstKey == null) {
- return null;
- }
- if (f.getComparator().compare(splitKey, firstKey) < 0) {
- return null;
- }
- }
- } finally {
- f.closeReader(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
- }
- }
-
- Path splitDir = new Path(getSplitsDir(hri), familyName);
- // A reference to the bottom half of the hsf store file.
- Reference r =
- top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
- // Add the referred-to regions name as a dot separated suffix.
- // See REF_NAME_REGEX regex above. The referred-to regions name is
- // up in the path of the passed in <code>f</code> -- parentdir is family,
- // then the directory above is the region name.
- String parentRegionName = regionInfoForFs.getEncodedName();
- // Write reference with same file id only with the other region name as
- // suffix and into the new region location (under same family).
- Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
- return r.write(fs, p);
- }
-
- // ===========================================================================
- // Merge Helpers
- // ===========================================================================
- /** @return {@link Path} to the temp directory used during merge operations */
- Path getMergesDir() {
- return new Path(getRegionDir(), REGION_MERGES_DIR);
- }
-
- Path getMergesDir(final HRegionInfo hri) {
- return new Path(getMergesDir(), hri.getEncodedName());
- }
-
- /**
- * Clean up any merge detritus that may have been left around from previous merge attempts.
- */
- void cleanupMergesDir() throws IOException {
- deleteDir(getMergesDir());
- }
-
- /**
- * Remove merged region
- * @param mergedRegion {@link HRegionInfo}
- * @throws IOException
- */
- void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
- Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
- if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
- throw new IOException("Failed delete of " + regionDir);
- }
- }
-
- /**
- * Create the region merges directory.
- * @throws IOException If merges dir already exists or we fail to create it.
- * @see HRegionFileSystem#cleanupMergesDir()
- */
- void createMergesDir() throws IOException {
- Path mergesdir = getMergesDir();
- if (fs.exists(mergesdir)) {
- LOG.info("The " + mergesdir
- + " directory exists. Hence deleting it to recreate it");
- if (!fs.delete(mergesdir, true)) {
- throw new IOException("Failed deletion of " + mergesdir
- + " before creating them again.");
- }
- }
- if (!fs.mkdirs(mergesdir))
- throw new IOException("Failed create of " + mergesdir);
- }
-
- /**
- * Write out a merge reference under the given merges directory. Package local
- * so it doesnt leak out of regionserver.
- * @param mergedRegion {@link HRegionInfo} of the merged region
- * @param familyName Column Family Name
- * @param f File to create reference.
- * @param mergedDir
- * @return Path to created reference.
- * @throws IOException
- */
- Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
- final StoreFile f, final Path mergedDir)
- throws IOException {
- Path referenceDir = new Path(new Path(mergedDir,
- mergedRegion.getEncodedName()), familyName);
- // A whole reference to the store file.
- Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
- // Add the referred-to regions name as a dot separated suffix.
- // See REF_NAME_REGEX regex above. The referred-to regions name is
- // up in the path of the passed in <code>f</code> -- parentdir is family,
- // then the directory above is the region name.
- String mergingRegionName = regionInfoForFs.getEncodedName();
- // Write reference with same file id only with the other region name as
- // suffix and into the new region location (under same family).
- Path p = new Path(referenceDir, f.getPath().getName() + "."
- + mergingRegionName);
- return r.write(fs, p);
- }
-
- /**
- * Commit a merged region, moving it from the merges temporary directory to
- * the proper location in the filesystem.
- * @param mergedRegionInfo merged region {@link HRegionInfo}
- * @throws IOException
- */
- void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
- Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
- Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
- // Move the tmp dir in the expected location
- if (mergedRegionTmpDir != null && fs.exists(mergedRegionTmpDir)) {
- if (!fs.rename(mergedRegionTmpDir, regionDir)) {
- throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
- + regionDir);
- }
- }
- }
-
- // ===========================================================================
- // Create/Open/Delete Helpers
- // ===========================================================================
- /**
- * Log the current state of the region
- * @param LOG log to output information
- * @throws IOException if an unexpected exception occurs
- */
- void logFileSystemState(final Log LOG) throws IOException {
- FSUtils.logFileSystemState(fs, this.getRegionDir(), LOG);
- }
-
- /**
- * @param hri
- * @return Content of the file we write out to the filesystem under a region
- * @throws IOException
- */
- private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
- return hri.toDelimitedByteArray();
- }
-
- /**
- * Create a {@link HRegionInfo} from the serialized version on-disk.
- * @param fs {@link FileSystem} that contains the Region Info file
- * @param regionDir {@link Path} to the Region Directory that contains the Info file
- * @return An {@link HRegionInfo} instance gotten from the Region Info file.
- * @throws IOException if an error occurred during file open/read operation.
- */
- public static HRegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
- throws IOException {
- FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
- try {
- return HRegionInfo.parseFrom(in);
- } finally {
- in.close();
- }
- }
-
- /**
- * Write the .regioninfo file on-disk.
- */
- private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
- final Path regionInfoFile, final byte[] content) throws IOException {
- // First check to get the permissions
- FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
- // Write the RegionInfo file content
- FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
- try {
- out.write(content);
- } finally {
- out.close();
- }
- }
-
- /**
- * Write out an info file under the stored region directory. Useful recovering mangled regions.
- * If the regionInfo already exists on-disk, then we fast exit.
- */
- void checkRegionInfoOnFilesystem() throws IOException {
- // Compose the content of the file so we can compare to length in filesystem. If not same,
- // rewrite it (it may have been written in the old format using Writables instead of pb). The
- // pb version is much shorter -- we write now w/o the toString version -- so checking length
- // only should be sufficient. I don't want to read the file every time to check if it pb
- // serialized.
- byte[] content = getRegionInfoFileContent(regionInfoForFs);
- try {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
-
- FileStatus status = fs.getFileStatus(regionInfoFile);
- if (status != null && status.getLen() == content.length) {
- // Then assume the content good and move on.
- // NOTE: that the length is not sufficient to define the the content matches.
- return;
- }
-
- LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
- if (!fs.delete(regionInfoFile, false)) {
- throw new IOException("Unable to remove existing " + regionInfoFile);
- }
- } catch (FileNotFoundException e) {
- LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfoForFs.getEncodedName() +
- " on table " + regionInfo.getTable());
- }
-
- // Write HRI to a file in case we need to recover hbase:meta
- writeRegionInfoOnFilesystem(content, true);
- }
-
- /**
- * Write out an info file under the region directory. Useful recovering mangled regions.
- * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
- */
- private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
- byte[] content = getRegionInfoFileContent(regionInfoForFs);
- writeRegionInfoOnFilesystem(content, useTempDir);
- }
-
- /**
- * Write out an info file under the region directory. Useful recovering mangled regions.
- * @param regionInfoContent serialized version of the {@link HRegionInfo}
- * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
- */
- private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent,
- final boolean useTempDir) throws IOException {
- Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
- if (useTempDir) {
- // Create in tmpDir and then move into place in case we crash after
- // create but before close. If we don't successfully close the file,
- // subsequent region reopens will fail the below because create is
- // registered in NN.
-
- // And then create the file
- Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
-
- // If datanode crashes or if the RS goes down just before the close is called while trying to
- // close the created regioninfo file in the .tmp directory then on next
- // creation we will be getting AlreadyCreatedException.
- // Hence delete and create the file if exists.
- if (FSUtils.isExists(fs, tmpPath)) {
- FSUtils.delete(fs, tmpPath, true);
- }
-
- // Write HRI to a file in case we need to recover hbase:meta
- writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
-
- // Move the created file to the original path
- if (fs.exists(tmpPath) && !rename(tmpPath, regionInfoFile)) {
- throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
- }
- } else {
- // Write HRI to a file in case we need to recover hbase:meta
- writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
- }
- }
-
- /**
- * Create a new Region on file-system.
- * @param conf the {@link Configuration} to use
- * @param fs {@link FileSystem} from which to add the region
- * @param tableDir {@link Path} to where the table is being stored
- * @param regionInfo {@link HRegionInfo} for region to be added
- * @throws IOException if the region creation fails due to a FileSystem exception.
- */
- public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
- final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
- HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
- Path regionDir = regionFs.getRegionDir();
-
- if (fs.exists(regionDir)) {
- LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
- throw new IOException("The specified region already exists on disk: " + regionDir);
- }
-
- // Create the region directory
- if (!createDirOnFileSystem(fs, conf, regionDir)) {
- LOG.warn("Unable to create the region directory: " + regionDir);
- throw new IOException("Unable to create region directory: " + regionDir);
- }
-
- // Write HRI to a file in case we need to recover hbase:meta
- regionFs.writeRegionInfoOnFilesystem(false);
- return regionFs;
- }
-
- /**
- * Open Region from file-system.
- * @param conf the {@link Configuration} to use
- * @param fs {@link FileSystem} from which to add the region
- * @param tableDir {@link Path} to where the table is being stored
- * @param regionInfo {@link HRegionInfo} for region to be added
- * @param readOnly True if you don't want to edit the region data
- * @throws IOException if the region creation fails due to a FileSystem exception.
- */
- public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
- final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo, boolean readOnly)
- throws IOException {
- HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
- Path regionDir = regionFs.getRegionDir();
-
- if (!fs.exists(regionDir)) {
- LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
- throw new IOException("The specified region do not exists on disk: " + regionDir);
- }
-
- if (!readOnly) {
- // Cleanup temporary directories
- regionFs.cleanupTempDir();
- regionFs.cleanupSplitsDir();
- regionFs.cleanupMergesDir();
-
- // if it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
- regionFs.checkRegionInfoOnFilesystem();
- }
-
- return regionFs;
- }
-
- /**
- * Remove the region from the table directory, archiving the region's hfiles.
- * @param conf the {@link Configuration} to use
- * @param fs {@link FileSystem} from which to remove the region
- * @param tableDir {@link Path} to where the table is being stored
- * @param regionInfo {@link HRegionInfo} for region to be deleted
- * @throws IOException if the request cannot be completed
- */
- public static void deleteRegionFromFileSystem(final Configuration conf,
- final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
- HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
- Path regionDir = regionFs.getRegionDir();
-
- if (!fs.exists(regionDir)) {
- LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
- return;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("DELETING region " + regionDir);
- }
-
- // Archive region
- Path rootDir = FSUtils.getRootDir(conf);
- HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
-
- // Delete empty region dir
- if (!fs.delete(regionDir, true)) {
- LOG.warn("Failed delete of " + regionDir);
- }
- }
-
- /**
- * Creates a directory. Assumes the user has already checked for this directory existence.
- * @param dir
- * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
- * whether the directory exists or not, and returns true if it exists.
- * @throws IOException
- */
- boolean createDir(Path dir) throws IOException {
- int i = 0;
- IOException lastIOE = null;
- do {
- try {
- return fs.mkdirs(dir);
- } catch (IOException ioe) {
- lastIOE = ioe;
- if (fs.exists(dir)) return true; // directory is present
- try {
- sleepBeforeRetry("Create Directory", i+1);
- } catch (InterruptedException e) {
- throw (InterruptedIOException)new InterruptedIOException().initCause(e);
- }
- }
- } while (++i <= hdfsClientRetriesNumber);
- throw new IOException("Exception in createDir", lastIOE);
- }
-
- /**
- * Renames a directory. Assumes the user has already checked for this directory existence.
- * @param srcpath
- * @param dstPath
- * @return true if rename is successful.
- * @throws IOException
- */
- boolean rename(Path srcpath, Path dstPath) throws IOException {
- IOException lastIOE = null;
- int i = 0;
- do {
- try {
- return fs.rename(srcpath, dstPath);
- } catch (IOException ioe) {
- lastIOE = ioe;
- if (!fs.exists(srcpath) && fs.exists(dstPath)) return true; // successful move
- // dir is not there, retry after some time.
- try {
- sleepBeforeRetry("Rename Directory", i+1);
- } catch (InterruptedException e) {
- throw (InterruptedIOException)new InterruptedIOException().initCause(e);
- }
- }
- } while (++i <= hdfsClientRetriesNumber);
-
- throw new IOException("Exception in rename", lastIOE);
- }
-
- /**
- * Deletes a directory. Assumes the user has already checked for this directory existence.
- * @param dir
- * @return true if the directory is deleted.
- * @throws IOException
- */
- boolean deleteDir(Path dir) throws IOException {
- IOException lastIOE = null;
- int i = 0;
- do {
- try {
- return fs.delete(dir, true);
- } catch (IOException ioe) {
- lastIOE = ioe;
- if (!fs.exists(dir)) return true;
- // dir is there, retry deleting after some time.
- try {
- sleepBeforeRetry("Delete Directory", i+1);
- } catch (InterruptedException e) {
- throw (InterruptedIOException)new InterruptedIOException().initCause(e);
- }
- }
- } while (++i <= hdfsClientRetriesNumber);
-
- throw new IOException("Exception in DeleteDir", lastIOE);
- }
-
- /**
- * sleeping logic; handles the interrupt exception.
- */
- private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException {
- sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
- }
-
- /**
- * Creates a directory for a filesystem and configuration object. Assumes the user has already
- * checked for this directory existence.
- * @param fs
- * @param conf
- * @param dir
- * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
- * whether the directory exists or not, and returns true if it exists.
- * @throws IOException
- */
- private static boolean createDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
- throws IOException {
- int i = 0;
- IOException lastIOE = null;
- int hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
- DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
- int baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
- DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
- do {
- try {
- return fs.mkdirs(dir);
- } catch (IOException ioe) {
- lastIOE = ioe;
- if (fs.exists(dir)) return true; // directory is present
- try {
- sleepBeforeRetry("Create Directory", i+1, baseSleepBeforeRetries, hdfsClientRetriesNumber);
- } catch (InterruptedException e) {
- throw (InterruptedIOException)new InterruptedIOException().initCause(e);
- }
- }
- } while (++i <= hdfsClientRetriesNumber);
-
- throw new IOException("Exception in createDir", lastIOE);
- }
-
- /**
- * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
- * for this to avoid re-looking for the integer values.
- */
- private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
- int hdfsClientRetriesNumber) throws InterruptedException {
- if (sleepMultiplier > hdfsClientRetriesNumber) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(msg + ", retries exhausted");
- }
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
- }
- Thread.sleep((long)baseSleepBeforeRetries * sleepMultiplier);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index c4bd849..e3b1a85 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.TagUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.conf.ConfigurationManager;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -127,7 +128,7 @@ public class HStore implements Store {
// This stores directory in the filesystem.
protected final HRegion region;
private final HColumnDescriptor family;
- private final HRegionFileSystem fs;
+ private final RegionFileSystem fs;
protected Configuration conf;
protected CacheConfig cacheConf;
private long lastCompactSize = 0;
@@ -335,7 +336,7 @@ public class HStore implements Store {
return this.fs.getFileSystem();
}
- public HRegionFileSystem getRegionFileSystem() {
+ public RegionFileSystem getRegionFileSystem() {
return this.fs;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
index 4a3f52f..3baa4d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.security.User;
* transaction, {@link #execute(Server, RegionServerServices)} to run the
* transaction and {@link #rollback(Server, RegionServerServices)} to cleanup if
* execute fails.
- *
+ *
* <p>Here is an example of how you would use this interface:
* <pre>
* RegionMergeTransactionFactory factory = new RegionMergeTransactionFactory(conf);
@@ -158,7 +158,7 @@ public interface RegionMergeTransaction {
* @param services
* @return <code>true</code> if the regions are mergeable else
* <code>false</code> if they are not (e.g. its already closed, etc.).
- * @throws IOException
+ * @throws IOException
*/
boolean prepare(RegionServerServices services) throws IOException;
@@ -217,7 +217,7 @@ public interface RegionMergeTransaction {
/**
* Register a listener for transaction preparation, execution, and possibly
* rollback phases.
- * <p>A listener can abort a transaction by throwing an exception.
+ * <p>A listener can abort a transaction by throwing an exception.
* @param listener the listener
* @return 'this' for chaining
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
index 9e7f97b..61c3eda 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl.LoggingProgressable;
import org.apache.hadoop.hbase.security.User;
@@ -200,7 +201,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
region_a.getRegionInfo().getRegionName());
if (regionAHasMergeQualifier ||
hasMergeQualifierInMeta(services, region_b.getRegionInfo().getRegionName())) {
- LOG.debug("Region " + (regionAHasMergeQualifier ?
+ LOG.debug("Region " + (regionAHasMergeQualifier ?
region_a.getRegionInfo().getRegionNameAsString()
: region_b.getRegionInfo().getRegionNameAsString())
+ " is not mergeable because it has merge qualifier in META");
@@ -556,23 +557,19 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
Map<byte[], List<StoreFile>> hstoreFilesOfRegionB)
throws IOException {
// Create reference file(s) of region A in mergdir
- HRegionFileSystem fs_a = this.region_a.getRegionFileSystem();
- for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA
- .entrySet()) {
+ RegionFileSystem fs_a = this.region_a.getRegionFileSystem();
+ for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA.entrySet()) {
String familyName = Bytes.toString(entry.getKey());
for (StoreFile storeFile : entry.getValue()) {
- fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
- this.mergesdir);
+ fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile, this.mergesdir);
}
}
// Create reference file(s) of region B in mergedir
- HRegionFileSystem fs_b = this.region_b.getRegionFileSystem();
- for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB
- .entrySet()) {
+ RegionFileSystem fs_b = this.region_b.getRegionFileSystem();
+ for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB.entrySet()) {
String familyName = Bytes.toString(entry.getKey());
for (StoreFile storeFile : entry.getValue()) {
- fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
- this.mergesdir);
+ fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile, this.mergesdir);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
index 21c117f..e4aee69 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
@@ -662,7 +663,7 @@ public class SplitTransactionImpl implements SplitTransaction {
LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " +
this.parent);
}
- HRegionFileSystem fs = this.parent.getRegionFileSystem();
+ RegionFileSystem fs = this.parent.getRegionFileSystem();
String familyName = Bytes.toString(family);
Path path_a =
fs.splitStoreFile(this.hri_a, familyName, sf, this.splitrow, false,
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index a8542de..4cde73d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.mob.MobUtils;
@@ -55,7 +56,6 @@ import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -778,9 +778,9 @@ public class RestoreSnapshotHelper {
FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
if (regionDirs == null) return null;
- List<HRegionInfo> regions = new ArrayList<HRegionInfo>(regionDirs.length);
- for (int i = 0; i < regionDirs.length; ++i) {
- HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
+ List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
+ for (FileStatus regionDir: regionDirs) {
+ HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
regions.add(hri);
}
LOG.debug("found " + regions.size() + " regions for table=" +
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index 0409441..dfb706b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
import org.apache.hadoop.hbase.mob.MobUtils;
@@ -51,7 +52,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -250,8 +250,8 @@ public final class SnapshotManifest {
boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
try {
// Open the RegionFS
- HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
- tableDir, regionInfo, true);
+ Path rootDir = null;
+ RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
monitor.rethrowException();
// 1. dump region meta info into the snapshot directory
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
index a5afb91..2c5d76a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
@@ -62,8 +62,7 @@ public final class SnapshotManifestV1 {
private SnapshotManifestV1() {
}
- static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
- HRegionFileSystem, Path> {
+ static class ManifestBuilder implements SnapshotManifest.RegionVisitor<RegionFileSystem, Path> {
private final Configuration conf;
private final Path snapshotDir;
private final FileSystem fs;
@@ -74,24 +73,24 @@ public final class SnapshotManifestV1 {
this.fs = fs;
}
- public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
- HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
- fs, snapshotDir, regionInfo);
+ public RegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
+ RegionFileSystem snapshotRegionFs = RegionFileSystem.open(conf, fs,
+ snapshotDir, regionInfo, true);
return snapshotRegionFs;
}
- public void regionClose(final HRegionFileSystem region) {
+ public void regionClose(final RegionFileSystem region) {
}
- public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
+ public Path familyOpen(final RegionFileSystem snapshotRegionFs, final byte[] familyName) {
Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
return familyDir;
}
- public void familyClose(final HRegionFileSystem region, final Path family) {
+ public void familyClose(final RegionFileSystem region, final Path family) {
}
- public void storeFile(final HRegionFileSystem region, final Path familyDir,
+ public void storeFile(final RegionFileSystem region, final Path familyDir,
final StoreFileInfo storeFile) throws IOException {
Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
boolean success = true;
@@ -126,7 +125,7 @@ public final class SnapshotManifestV1 {
completionService.submit(new Callable<SnapshotRegionManifest>() {
@Override
public SnapshotRegionManifest call() throws IOException {
- HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
+ HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
return buildManifestFromDisk(conf, fs, snapshotDir, hri);
}
});
@@ -156,8 +155,7 @@ public final class SnapshotManifestV1 {
static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf,
final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
- HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
- tableDir, regionInfo, true);
+ RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, tableDir, regionInfo, true);
SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
// 1. dump region meta info into the snapshot directory
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index e1a3fa2..2f34291 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -111,7 +111,9 @@ import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -120,7 +122,6 @@ import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -236,6 +237,8 @@ public class HBaseFsck extends Configured implements Closeable {
// successful
private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
+ private final MasterFileSystem mfs;
+
/***********
* Options
***********/
@@ -348,8 +351,18 @@ public class HBaseFsck extends Configured implements Closeable {
public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
ZooKeeperConnectionException, IOException, ClassNotFoundException {
super(conf);
+ // make a copy, just to be sure we're not overriding someone else's config
+ setConf(HBaseConfiguration.create(getConf()));
+ // disable blockcache for tool invocation, see HBASE-10500
+ getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
+ // Disable usage of meta replicas in hbck
+ getConf().setBoolean(HConstants.USE_META_REPLICAS, false);
+
+ mfs = MasterFileSystem.open(getConf(), false);
+
errors = getErrorReporter(getConf());
this.executor = exec;
+
lockFileRetryCounterFactory = new RetryCounterFactory(
getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
getConf().getInt(
@@ -1075,7 +1088,6 @@ public class HBaseFsck extends Configured implements Closeable {
private void offlineReferenceFileRepair() throws IOException {
clearState();
LOG.info("Validating mapping using HDFS state");
- final MasterFileSystem mfs = MasterFileSystem.open(getConf(), false);
mfs.visitStoreFiles(new StoreFileVisitor() {
@Override
public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
@@ -1179,7 +1191,7 @@ public class HBaseFsck extends Configured implements Closeable {
}
FileSystem fs = FileSystem.get(getConf());
- HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+ HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
LOG.debug("HRegionInfo read: " + hri.toString());
hbi.hdfsEntry.hri = hri;
}
@@ -1412,7 +1424,7 @@ public class HBaseFsck extends Configured implements Closeable {
Collections.<WALActionsListener>singletonList(new MetricsWAL()),
"hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
- HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
+ HRegion meta = HRegion.createHRegion(c, rootdir, metaDescriptor, metaHRI, wal);
MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
return meta;
}
@@ -2349,8 +2361,7 @@ public class HBaseFsck extends Configured implements Closeable {
LOG.warn(hri + " start and stop keys are in the range of " + region
+ ". The region might not be cleaned up from hdfs when region " + region
+ " split failed. Hence deleting from hdfs.");
- HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
- regionDir.getParent(), hri);
+ RegionFileSystem.destroy(getConf(), fs, regionDir.getParent(), hri);
return;
}
}
@@ -2612,7 +2623,7 @@ public class HBaseFsck extends Configured implements Closeable {
Path src = cf.getPath();
Path dst = new Path(targetRegionDir, src.getName());
- if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
+ if (src.getName().equals(LegacyLayout.REGION_INFO_FILE)) {
// do not copy the old .regioninfo file.
continue;
}
@@ -4135,7 +4146,7 @@ public class HBaseFsck extends Configured implements Closeable {
try {
LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
- Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);
+ Path regioninfoFile = new Path(he.hdfsRegionDir, LegacyLayout.REGION_INFO_FILE);
boolean regioninfoFileExists = fs.exists(regioninfoFile);
if (!regioninfoFileExists) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index eaf8d54..a07fed9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -191,7 +191,7 @@ public class HBaseFsckRepair {
HRegionInfo hri, HTableDescriptor htd) throws IOException {
// Create HRegion
Path root = FSUtils.getRootDir(conf);
- HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
+ HRegion region = HRegion.createHRegion(conf, root, htd, hri, null);
// Close the new region to flush to disk. Close log file too.
region.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index a936fc2..4fe39ff 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -170,7 +170,7 @@ public abstract class ModifyRegionUtils {
// unless I pass along via the conf.
Configuration confForWAL = new Configuration(conf);
confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
- HRegion region = HRegion.createHRegion(newRegion, rootDir, conf, hTableDescriptor, null, false);
+ HRegion region = HRegion.createHRegion(conf, rootDir, hTableDescriptor, newRegion, null, false);
try {
// 2. Custom user code to interact with the created region
if (task != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index a6f70c3..cbc57dc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.NoServerForRegionException;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -412,7 +412,7 @@ public class RegionSplitter {
* Alternative getCurrentNrHRS which is no longer available.
* @param connection
* @return Rough count of regionservers out on cluster.
- * @throws IOException
+ * @throws IOException
*/
private static int getRegionServerCount(final Connection connection) throws IOException {
try (Admin admin = connection.getAdmin()) {
@@ -727,8 +727,8 @@ public class RegionSplitter {
if (sk.length == 0)
sk = splitAlgo.firstRow();
- HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
- connection.getConfiguration(), fs, tableDir, hri, true);
+ RegionFileSystem regionFs = RegionFileSystem.open(
+ connection.getConfiguration(), fs, tableDir, hri, true);
// Check every Column Family for that region -- check does not have references.
boolean refFound = false;
@@ -767,7 +767,7 @@ public class RegionSplitter {
* @param conf
* @param tableName
* @return A Pair where first item is table dir and second is the split file.
- * @throws IOException
+ * @throws IOException
*/
private static Pair<Path, Path> getTableDirAndSplitFile(final Configuration conf,
final TableName tableName)
@@ -785,7 +785,7 @@ public class RegionSplitter {
getTableDirAndSplitFile(connection.getConfiguration(), tableName);
Path tableDir = tableDirAndSplitFile.getFirst();
Path splitFile = tableDirAndSplitFile.getSecond();
-
+
FileSystem fs = tableDir.getFileSystem(connection.getConfiguration());
// Using strings because (new byte[]{0}).equals(new byte[]{0}) == false
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 8335f4f..30ee2d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1815,7 +1815,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
*/
public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
throws IOException {
- return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal);
+ return HRegion.createHRegion(getConfiguration(), getDataTestDir(), desc, info, wal);
}
/**
@@ -1828,7 +1828,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* @param families
* @throws IOException
* @return A region on which you must call
- {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+ * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
* @deprecated use
* {@link #createLocalHRegion(TableName, byte[], byte[], boolean, Durability, WAL, byte[]...)}
*/
@@ -2347,7 +2347,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
final Configuration conf, final HTableDescriptor htd, boolean initialize)
throws IOException {
WAL wal = createWal(conf, rootDir, info);
- return HRegion.createHRegion(info, rootDir, conf, htd, wal, initialize);
+ return HRegion.createHRegion(conf, rootDir, htd, info, wal, initialize);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
index 0217b41..6c2c882 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
@@ -93,11 +94,9 @@ public class TestIOFencing {
volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
- @SuppressWarnings("deprecation")
- public CompactionBlockerRegion(Path tableDir, WAL log,
- FileSystem fs, Configuration confParam, HRegionInfo info,
- HTableDescriptor htd, RegionServerServices rsServices) {
- super(tableDir, log, fs, confParam, info, htd, rsServices);
+ public CompactionBlockerRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+ final WAL wal, final RegionServerServices rsServices) {
+ super(rfs, htd, wal, rsServices);
}
public void stopCompactions() {
@@ -154,11 +153,11 @@ public class TestIOFencing {
*/
public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
- public BlockCompactionsInPrepRegion(Path tableDir, WAL log,
- FileSystem fs, Configuration confParam, HRegionInfo info,
- HTableDescriptor htd, RegionServerServices rsServices) {
- super(tableDir, log, fs, confParam, info, htd, rsServices);
+ public BlockCompactionsInPrepRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+ final WAL wal, final RegionServerServices rsServices) {
+ super(rfs, htd, wal, rsServices);
}
+
@Override
protected void doRegionCompactionPrep() throws IOException {
compactionsWaiting.countDown();
@@ -177,11 +176,11 @@ public class TestIOFencing {
* entry to go the WAL before blocking, but blocks afterwards
*/
public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
- public BlockCompactionsInCompletionRegion(Path tableDir, WAL log,
- FileSystem fs, Configuration confParam, HRegionInfo info,
- HTableDescriptor htd, RegionServerServices rsServices) {
- super(tableDir, log, fs, confParam, info, htd, rsServices);
+ public BlockCompactionsInCompletionRegion(final RegionFileSystem rfs,
+ final HTableDescriptor htd, final WAL wal, final RegionServerServices rsServices) {
+ super(rfs, htd, wal, rsServices);
}
+
@Override
protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
return new BlockCompactionsInCompletionHStore(this, family, this.conf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 53de8a0..a5b0002 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -31,11 +31,14 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.ClientTests;
@@ -302,13 +305,14 @@ public class TestRestoreSnapshotFromClient {
private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
- Set<String> families = new HashSet<String>();
- Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
- for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
- for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
- families.add(familyDir.getName());
+ final Set<String> families = new HashSet<String>();
+ mfs.visitStoreFiles(tableName, new StoreFileVisitor() {
+ @Override
+ public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+ throws IOException {
+ families.add(family);
}
- }
+ });
return families;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index 02bb0d3..fa4497f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -153,7 +154,7 @@ public class TestRegionObserverScannerOpenHook {
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
Path path = new Path(DIR + callingMethod);
WAL wal = HBaseTestingUtility.createWal(conf, path, info);
- HRegion r = HRegion.createHRegion(info, path, conf, htd, wal);
+ HRegion r = HRegion.createHRegion(conf, path, htd, info, wal);
// this following piece is a hack. currently a coprocessorHost
// is secretly loaded at OpenRegionHandler. we don't really
// start a region server here, so just manually create cphost
@@ -220,11 +221,9 @@ public class TestRegionObserverScannerOpenHook {
public static class CompactionCompletionNotifyingRegion extends HRegion {
private static volatile CountDownLatch compactionStateChangeLatch = null;
- @SuppressWarnings("deprecation")
- public CompactionCompletionNotifyingRegion(Path tableDir, WAL log,
- FileSystem fs, Configuration confParam, HRegionInfo info,
- HTableDescriptor htd, RegionServerServices rsServices) {
- super(tableDir, log, fs, confParam, info, htd, rsServices);
+ public CompactionCompletionNotifyingRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+ final WAL wal, final RegionServerServices rsServices) {
+ super(rfs, htd, wal, rsServices);
}
public CountDownLatch getCompactionStateChangeLatch() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 26e5897..89c8dfc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -26,10 +26,12 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
@@ -125,26 +127,24 @@ public abstract class TableSnapshotInputFormatTestBase {
testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
- Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
- for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) {
- for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
- for (FileStatus status : fs.listStatus(storeDir)) {
- System.out.println(status.getPath());
- if (StoreFileInfo.isValid(status)) {
- Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
- tableName, regionDir.getName(), storeDir.getName());
-
- Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
- // assert back references directory is empty
- assertFalse("There is a back reference in " + path, fs.exists(path));
-
- path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
- // assert back references directory is empty
- assertFalse("There is a back reference in " + path, fs.exists(path));
- }
- }
+ UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(tableName,
+ new StoreFileVisitor() {
+ @Override
+ public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+ throws IOException {
+ Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
+ region.getTable(), region.getEncodedName(), family);
+
+ // assert back references directory is empty
+ Path storeDir = storeFile.getPath().getParent();
+ Path path = HFileLink.getBackReferencesDir(storeDir, storeFile.getPath().getName());
+ assertFalse("There is a back reference in " + path, fs.exists(path));
+
+ // assert back references directory is empty
+ path = HFileLink.getBackReferencesDir(archiveStoreDir, storeFile.getPath().getName());
+ assertFalse("There is a back reference in " + path, fs.exists(path));
}
- }
+ });
} finally {
UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
UTIL.deleteTable(tableName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/457ebd7b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index e9e8bdc..b7e980b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -41,11 +41,13 @@ import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -199,19 +201,20 @@ public class MasterProcedureTestingUtility {
}
public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName,
- final String family) throws IOException {
+ final String deletedFamily) throws IOException {
// verify htd
HTableDescriptor htd = master.getTableDescriptors().get(tableName);
assertTrue(htd != null);
- assertFalse(htd.hasFamily(family.getBytes()));
+ assertFalse(htd.hasFamily(deletedFamily.getBytes()));
// verify fs
- final FileSystem fs = master.getMasterFileSystem().getFileSystem();
- final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
- for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
- final Path familyDir = new Path(regionDir, family);
- assertFalse(family + " family dir should not exist", fs.exists(familyDir));
- }
+ master.getMasterFileSystem().visitStoreFiles(tableName, new StoreFileVisitor() {
+ @Override
+ public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+ throws IOException {
+ assertFalse(family + " family dir should not exist", family.equals(deletedFamily));
+ }
+ });
}
public static void validateColumnFamilyModification(final HMaster master,