You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/03/25 12:08:56 UTC
svn commit: r1460611 [1/2] - in /hbase/branches/0.95/hbase-server/src:
main/java/org/apache/hadoop/hbase/master/snapshot/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/snapshot/
main/java/org/apache/hadoop/hbase/util...
Author: mbertozzi
Date: Mon Mar 25 11:08:56 2013
New Revision: 1460611
URL: http://svn.apache.org/r1460611
Log:
HBASE-7807 Introduce HRegionFileSystem and move region fs related code
Added:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Mon Mar 25 11:08:56 2013
@@ -34,9 +34,11 @@ import org.apache.hadoop.hbase.errorhand
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -80,6 +82,8 @@ public class DisabledTableSnapshotHandle
try {
timeoutInjector.start();
+ Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
+
// 1. get all the regions hosting this table.
// extract each pair to separate lists
@@ -95,14 +99,15 @@ public class DisabledTableSnapshotHandle
+ ClientSnapshotDescriptionUtils.toString(snapshot));
for (HRegionInfo regionInfo : regions) {
// 2.1 copy the regionInfo files to the snapshot
- Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
- regionInfo.getEncodedName());
- HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
+ HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
+ snapshotDir, regionInfo);
+
// check for error for each region
monitor.rethrowException();
// 2.2 for each region, copy over its recovered.edits directory
Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+ Path snapshotRegionDir = regionFs.getRegionDir();
new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
monitor.rethrowException();
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java Mon Mar 25 11:08:56 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.catalog.M
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.exceptions.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
@@ -166,21 +167,16 @@ public final class MasterSnapshotVerifie
snapshot);
}
// make sure we have the region info in the snapshot
- Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
+ Path regionInfo = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
// make sure the file exists
if (!fs.exists(regionInfo)) {
throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
}
- FSDataInputStream in = fs.open(regionInfo);
- HRegionInfo found;
- try {
- found = HRegionInfo.parseFrom(in);
- if (!region.equals(found)) {
- throw new CorruptedSnapshotException("Found region info (" + found
- + ") doesn't match expected region:" + region, snapshot);
- }
- } finally {
- in.close();
+
+ HRegionInfo found = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+ if (!region.equals(found)) {
+ throw new CorruptedSnapshotException("Found region info (" + found
+ + ") doesn't match expected region:" + region, snapshot);
}
// make sure we have the expected recovered edits files
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java Mon Mar 25 11:08:56 2013
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.HDFSBlock
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.mapreduce.JobUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
@@ -194,13 +195,13 @@ public class CompactionTool extends Conf
private static HRegion loadRegion(final FileSystem fs, final Configuration conf,
final HTableDescriptor htd, final Path regionDir) throws IOException {
Path rootDir = regionDir.getParent().getParent();
- HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+ HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
return HRegion.createHRegion(hri, rootDir, conf, htd, null, false, true);
}
}
private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
- Path regionInfo = new Path(path, HRegion.REGIONINFO_FILE);
+ Path regionInfo = new Path(path, HRegionFileSystem.REGION_INFO_FILE);
return fs.exists(regionInfo);
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Mon Mar 25 11:08:56 2013
@@ -64,8 +64,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -126,6 +124,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CancelableProgressable;
@@ -221,11 +220,6 @@ public class HRegion implements HeapSize
// TODO: account for each registered handler in HeapSize computation
private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
- /**
- * Temporary subdirectory of the region directory used for compaction output.
- */
- public static final String REGION_TEMP_SUBDIR = ".tmp";
-
//These variable are just used for getting data out of the region, to test on
//client side
// private int numStores = 0;
@@ -249,16 +243,11 @@ public class HRegion implements HeapSize
//How long operations were blocked by a memstore over highwater.
final Counter updatesBlockedMs = new Counter();
- /**
- * The directory for the table this region is part of.
- * This directory contains the directory for this region.
- */
- private final Path tableDir;
-
private final HLog log;
- private final FileSystem fs;
+ private final HRegionFileSystem fs;
private final Configuration conf;
private final Configuration baseConf;
+ private final KeyValue.KVComparator comparator;
private final int rowLockWaitDuration;
static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
@@ -284,10 +273,6 @@ public class HRegion implements HeapSize
static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L;
final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool();
- private final HRegionInfo regionInfo;
- private final Path regiondir;
- KeyValue.KVComparator comparator;
-
private final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;
/**
@@ -390,10 +375,6 @@ public class HRegion implements HeapSize
// Coprocessor host
private RegionCoprocessorHost coprocessorHost;
- /**
- * Name of the region info file that resides just under the region directory.
- */
- public final static String REGIONINFO_FILE = ".regioninfo";
private HTableDescriptor htableDescriptor = null;
private RegionSplitPolicy splitPolicy;
@@ -402,10 +383,9 @@ public class HRegion implements HeapSize
private final boolean deferredLogSyncDisabled;
/**
- * HRegion constructor. his constructor should only be used for testing and
+ * HRegion constructor. This constructor should only be used for testing and
* extensions. Instances of HRegion should be instantiated with the
- * {@link HRegion#newHRegion(Path, HLog, FileSystem, Configuration, HRegionInfo, HTableDescriptor, RegionServerServices)} method.
- *
+ * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.
*
* @param tableDir qualified path of directory where region should be located,
* usually the table directory.
@@ -420,23 +400,48 @@ public class HRegion implements HeapSize
* @param confParam is global configuration settings.
* @param regionInfo - HRegionInfo that describes the region
* is new), then read them from the supplied path.
+ * @param htd the table descriptor
* @param rsServices reference to {@link RegionServerServices} or null
+ */
+ @Deprecated
+ public HRegion(final Path tableDir, final HLog log, final FileSystem fs,
+ final Configuration confParam, final HRegionInfo regionInfo,
+ final HTableDescriptor htd, final RegionServerServices rsServices) {
+ this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo),
+ log, confParam, htd, rsServices);
+ }
+
+ /**
+ * HRegion constructor. This constructor should only be used for testing and
+ * extensions. Instances of HRegion should be instantiated with the
+ * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.
*
- * @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, HRegionInfo, HTableDescriptor, RegionServerServices)
+ * @param fs is the filesystem.
+ * @param log The HLog is the outbound log for any updates to the HRegion
+ * (There's a single HLog for all the HRegions on a single HRegionServer.)
+ * The log file is a logfile from the previous execution that's
+ * custom-computed for this HRegion. The HRegionServer computes and sorts the
+ * appropriate log info for this HRegion. If there is a previous log file
+ * (implying that the HRegion has been written-to before), then read it from
+ * the supplied path.
+ * @param confParam is global configuration settings.
+ * @param htd the table descriptor
+ * @param rsServices reference to {@link RegionServerServices} or null
*/
- public HRegion(Path tableDir, HLog log, FileSystem fs,
- Configuration confParam, final HRegionInfo regionInfo,
- final HTableDescriptor htd, RegionServerServices rsServices) {
+ public HRegion(final HRegionFileSystem fs, final HLog log, final Configuration confParam,
+ final HTableDescriptor htd, final RegionServerServices rsServices) {
if (htd == null) {
throw new IllegalArgumentException("Need table descriptor");
}
- this.tableDir = tableDir;
- this.comparator = regionInfo.getComparator();
- this.log = log;
- this.fs = fs;
+
if (confParam instanceof CompoundConfiguration) {
throw new IllegalArgumentException("Need original base configuration");
}
+
+ this.comparator = fs.getRegionInfo().getComparator();
+ this.log = log;
+ this.fs = fs;
+
// 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
this.baseConf = confParam;
this.conf = new CompoundConfiguration()
@@ -447,14 +452,10 @@ public class HRegion implements HeapSize
DEFAULT_ROWLOCK_WAIT_DURATION);
this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, false);
- this.regionInfo = regionInfo;
this.htableDescriptor = htd;
this.rsServices = rsServices;
- this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
- 10 * 1000);
- String encodedNameStr = this.regionInfo.getEncodedName();
+ this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
setHTableSpecificConf();
- this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
this.busyWaitDuration = conf.getLong(
@@ -554,8 +555,8 @@ public class HRegion implements HeapSize
}
}
- private long initializeRegionInternals(final CancelableProgressable reporter, MonitoredTask status)
- throws IOException, UnsupportedEncodingException {
+ private long initializeRegionInternals(final CancelableProgressable reporter,
+ final MonitoredTask status) throws IOException, UnsupportedEncodingException {
if (coprocessorHost != null) {
status.setStatus("Running coprocessor pre-open hook");
coprocessorHost.preOpen();
@@ -563,20 +564,58 @@ public class HRegion implements HeapSize
// Write HRI to a file in case we need to recover .META.
status.setStatus("Writing region info on filesystem");
- checkRegioninfoOnFilesystem();
+ fs.checkRegionInfoOnFilesystem();
// Remove temporary data left over from old regions
status.setStatus("Cleaning up temporary data from old regions");
- cleanupTmpDir();
+ fs.cleanupTempDir();
+
+ // Initialize all the HStores
+ status.setStatus("Initializing all the Stores");
+ long maxSeqId = initializeRegionStores(reporter, status);
+
+ status.setStatus("Cleaning up detritus from prior splits");
+ // Get rid of any splits or merges that were lost in-progress. Clean out
+ // these directories here on open. We may be opening a region that was
+ // being split but we crashed in the middle of it all.
+ SplitTransaction.cleanupAnySplitDetritus(this);
+ fs.cleanupMergesDir();
+
+ this.writestate.setReadOnly(this.htableDescriptor.isReadOnly());
+ this.writestate.flushRequested = false;
+ this.writestate.compacting = 0;
+
+ // Initialize split policy
+ this.splitPolicy = RegionSplitPolicy.create(this, conf);
+
+ this.lastFlushTime = EnvironmentEdgeManager.currentTimeMillis();
+ // Use maximum of log sequenceid or that which was found in stores
+ // (particularly if no recovered edits, seqid will be -1).
+ long nextSeqid = maxSeqId + 1;
+ LOG.info("Onlined " + this.toString() + "; next sequenceid=" + nextSeqid);
+ // A region can be reopened if failed a split; reset flags
+ this.closing.set(false);
+ this.closed.set(false);
+
+ if (coprocessorHost != null) {
+ status.setStatus("Running coprocessor post-open hooks");
+ coprocessorHost.postOpen();
+ }
+
+ status.markComplete("Region opened successfully");
+ return nextSeqid;
+ }
+
+ private long initializeRegionStores(final CancelableProgressable reporter, MonitoredTask status)
+ throws IOException, UnsupportedEncodingException {
// Load in all the HStores.
//
// Context: During replay we want to ensure that we do not lose any data. So, we
// have to be conservative in how we replay logs. For each store, we calculate
// the maxSeqId up to which the store was flushed. And, skip the edits which
// is equal to or lower than maxSeqId for each store.
- Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
- Bytes.BYTES_COMPARATOR);
+ Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
long maxSeqId = -1;
// initialized to -1 so that we pick up MemstoreTS from column families
long maxMemstoreTS = -1;
@@ -585,7 +624,7 @@ public class HRegion implements HeapSize
// initialize the thread pool for opening stores in parallel.
ThreadPoolExecutor storeOpenerThreadPool =
getStoreOpenAndCloseThreadPool(
- "StoreOpenerThread-" + this.regionInfo.getRegionNameAsString());
+ "StoreOpenerThread-" + this.getRegionNameAsString());
CompletionService<HStore> completionService =
new ExecutorCompletionService<HStore>(storeOpenerThreadPool);
@@ -594,7 +633,7 @@ public class HRegion implements HeapSize
status.setStatus("Instantiating store for column family " + family);
completionService.submit(new Callable<HStore>() {
public HStore call() throws IOException {
- return instantiateHStore(tableDir, family);
+ return instantiateHStore(getTableDir(), family);
}
});
}
@@ -629,7 +668,7 @@ public class HRegion implements HeapSize
mvcc.initialize(maxMemstoreTS + 1);
// Recover any edits if available.
maxSeqId = Math.max(maxSeqId, replayRecoveredEditsIfAny(
- this.regiondir, maxSeqIdInStores, reporter, status));
+ this.getRegionDir(), maxSeqIdInStores, reporter, status));
status.setStatus("Cleaning up detritus from prior splits");
// Get rid of any splits or merges that were lost in-progress. Clean out
@@ -638,7 +677,7 @@ public class HRegion implements HeapSize
SplitTransaction.cleanupAnySplitDetritus(this);
RegionMergeTransaction.cleanupMergeDir(this.getFilesystem(),
RegionMergeTransaction.getMergeDir(this));
- FSUtils.deleteDirectory(this.fs, new Path(regiondir, MERGEDIR));
+ FSUtils.deleteDirectory(this.getFilesystem(), new Path(this.getRegionDir(), MERGEDIR));
this.writestate.setReadOnly(this.htableDescriptor.isReadOnly());
this.writestate.flushRequested = false;
@@ -764,117 +803,9 @@ public class HRegion implements HeapSize
return this.memstoreSize.getAndAdd(memStoreSize);
}
- /**
- * Write out an info file under the stored region directory. Useful recovering mangled regions.
- * @throws IOException
- */
- private void checkRegioninfoOnFilesystem() throws IOException {
- checkRegioninfoOnFilesystem(this.regiondir);
- }
-
- /**
- * Write out an info file under the region directory. Useful recovering mangled regions.
- * @param regiondir directory under which to write out the region info
- * @throws IOException
- */
- private void checkRegioninfoOnFilesystem(Path regiondir) throws IOException {
- writeRegioninfoOnFilesystem(regionInfo, regiondir, getFilesystem(), conf);
- }
-
- /**
- * Write out an info file under the region directory. Useful recovering mangled regions. If the
- * regioninfo already exists on disk and there is information in the file, then we fast exit.
- * @param regionInfo information about the region
- * @param regiondir directory under which to write out the region info
- * @param fs {@link FileSystem} on which to write the region info
- * @param conf {@link Configuration} from which to extract specific file locations
- * @throws IOException on unexpected error.
- */
- public static void writeRegioninfoOnFilesystem(HRegionInfo regionInfo, Path regiondir,
- FileSystem fs, Configuration conf) throws IOException {
- Path regioninfoPath = new Path(regiondir, REGIONINFO_FILE);
- // Compose the content of the file so we can compare to length in filesystem. If not same,
- // rewrite it (it may have been written in the old format using Writables instead of pb). The
- // pb version is much shorter -- we write now w/o the toString version -- so checking length
- // only should be sufficient. I don't want to read the file every time to check if it pb
- // serialized.
- byte[] content = getDotRegionInfoFileContent(regionInfo);
- boolean exists = fs.exists(regioninfoPath);
- FileStatus status = exists ? fs.getFileStatus(regioninfoPath) : null;
- if (status != null && status.getLen() == content.length) {
- // Then assume the content good and move on.
- return;
- }
- // Create in tmpdir and then move into place in case we crash after
- // create but before close. If we don't successfully close the file,
- // subsequent region reopens will fail the below because create is
- // registered in NN.
-
- // First check to get the permissions
- FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
-
- // And then create the file
- Path tmpPath = new Path(getTmpDir(regiondir), REGIONINFO_FILE);
-
- // If datanode crashes or if the RS goes down just before the close is called while trying to
- // close the created regioninfo file in the .tmp directory then on next
- // creation we will be getting AlreadyCreatedException.
- // Hence delete and create the file if exists.
- if (FSUtils.isExists(fs, tmpPath)) {
- FSUtils.delete(fs, tmpPath, true);
- }
-
- FSDataOutputStream out = FSUtils.create(fs, tmpPath, perms);
- try {
- // We used to write out this file as serialized Writable followed by '\n\n' and then the
- // toString of the HRegionInfo but now we just write out the pb serialized bytes so we can
- // for sure tell whether the content has been pb'd or not just by looking at file length; the
- // pb version will be shorter.
- out.write(content);
- } finally {
- out.close();
- }
- if (exists) {
- LOG.info("Rewriting .regioninfo file at " + regioninfoPath);
- if (!fs.delete(regioninfoPath, false)) {
- throw new IOException("Unable to remove existing " + regioninfoPath);
- }
- }
- if (!fs.rename(tmpPath, regioninfoPath)) {
- throw new IOException("Unable to rename " + tmpPath + " to " + regioninfoPath);
- }
- }
-
- /**
- * @param hri
- * @return Content of the file we write out to the filesystem under a region
- * @throws IOException
- */
- private static byte [] getDotRegionInfoFileContent(final HRegionInfo hri) throws IOException {
- return hri.toDelimitedByteArray();
- }
-
- /**
- * @param fs
- * @param dir
- * @return An HRegionInfo instance gotten from the <code>.regioninfo</code> file under region dir
- * @throws IOException
- */
- public static HRegionInfo loadDotRegionInfoFileContent(final FileSystem fs, final Path dir)
- throws IOException {
- Path regioninfo = new Path(dir, HRegion.REGIONINFO_FILE);
- if (!fs.exists(regioninfo)) throw new FileNotFoundException(regioninfo.toString());
- FSDataInputStream in = fs.open(regioninfo);
- try {
- return HRegionInfo.parseFrom(in);
- } finally {
- in.close();
- }
- }
-
/** @return a HRegionInfo object for this region */
public HRegionInfo getRegionInfo() {
- return this.regionInfo;
+ return this.fs.getRegionInfo();
}
/**
@@ -1055,8 +986,7 @@ public class HRegion implements HeapSize
if (!stores.isEmpty()) {
// initialize the thread pool for closing stores in parallel.
ThreadPoolExecutor storeCloserThreadPool =
- getStoreOpenAndCloseThreadPool("StoreCloserThread-"
- + this.regionInfo.getRegionNameAsString());
+ getStoreOpenAndCloseThreadPool("StoreCloserThread-" + this.getRegionNameAsString());
CompletionService<Collection<StoreFile>> completionService =
new ExecutorCompletionService<Collection<StoreFile>>(storeCloserThreadPool);
@@ -1168,27 +1098,27 @@ public class HRegion implements HeapSize
/** @return start key for region */
public byte [] getStartKey() {
- return this.regionInfo.getStartKey();
+ return this.getRegionInfo().getStartKey();
}
/** @return end key for region */
public byte [] getEndKey() {
- return this.regionInfo.getEndKey();
+ return this.getRegionInfo().getEndKey();
}
/** @return region id */
public long getRegionId() {
- return this.regionInfo.getRegionId();
+ return this.getRegionInfo().getRegionId();
}
/** @return region name */
public byte [] getRegionName() {
- return this.regionInfo.getRegionName();
+ return this.getRegionInfo().getRegionName();
}
/** @return region name as string for logging */
public String getRegionNameAsString() {
- return this.regionInfo.getRegionNameAsString();
+ return this.getRegionInfo().getRegionNameAsString();
}
/** @return HTableDescriptor for this region */
@@ -1214,22 +1144,16 @@ public class HRegion implements HeapSize
/** @return region directory Path */
public Path getRegionDir() {
- return this.regiondir;
+ return fs.getRegionDir();
}
- /**
- * Computes the Path of the HRegion
- *
- * @param tabledir qualified path for table
- * @param name ENCODED region name
- * @return Path of HRegion directory
- */
- public static Path getRegionDir(final Path tabledir, final String name) {
- return new Path(tabledir, name);
+ /** @return {@link FileSystem} being used by this region */
+ public FileSystem getFilesystem() {
+ return fs.getFileSystem();
}
- /** @return FileSystem being used by this region */
- public FileSystem getFilesystem() {
+ /** @return the {@link HRegionFileSystem} used by this region */
+ public HRegionFileSystem getRegionFileSystem() {
return this.fs;
}
@@ -1264,27 +1188,12 @@ public class HRegion implements HeapSize
void doRegionCompactionPrep() throws IOException {
}
- /*
- * Removes the temporary directory for this Store.
- */
- private void cleanupTmpDir() throws IOException {
- FSUtils.deleteDirectory(this.fs, getTmpDir());
- }
-
/**
* Get the temporary directory for this region. This directory
* will have its contents removed when the region is reopened.
*/
Path getTmpDir() {
- return getTmpDir(getRegionDir());
- }
-
- /**
- * Get the temporary directory for the specified region. This directory
- * will have its contents removed when the region is reopened.
- */
- static Path getTmpDir(Path regionDir) {
- return new Path(regionDir, REGION_TEMP_SUBDIR);
+ return fs.getTempDir();
}
void triggerMajorCompaction() {
@@ -1585,9 +1494,9 @@ public class HRegion implements HeapSize
mvcc.advanceMemstore(w);
if (wal != null) {
- Long startSeqId = wal.startCacheFlush(this.regionInfo.getEncodedNameAsBytes());
+ Long startSeqId = wal.startCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
if (startSeqId == null) {
- status.setStatus("Flush will not be started for [" + this.regionInfo.getEncodedName()
+ status.setStatus("Flush will not be started for [" + this.getRegionInfo().getEncodedName()
+ "] - WAL is going away");
return false;
}
@@ -1657,7 +1566,7 @@ public class HRegion implements HeapSize
// exceptions -- e.g. HBASE-659 was about an NPE -- so now we catch
// all and sundry.
if (wal != null) {
- wal.abortCacheFlush(this.regionInfo.getEncodedNameAsBytes());
+ wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
}
DroppedSnapshotException dse = new DroppedSnapshotException("region: " +
Bytes.toStringBinary(getRegionName()));
@@ -1668,7 +1577,7 @@ public class HRegion implements HeapSize
// If we get to here, the HStores have been written.
if (wal != null) {
- wal.completeCacheFlush(this.regionInfo.getEncodedNameAsBytes());
+ wal.completeCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
}
// Update the last flushed sequence id for region
@@ -2283,7 +2192,7 @@ public class HRegion implements HeapSize
// STEP 5. Append the edit to WAL. Do not sync wal.
// -------------------------
Mutation first = batchOp.operations[firstIndex].getFirst();
- txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
+ txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(),
walEdit, first.getClusterId(), now, this.htableDescriptor);
// -------------------------------
@@ -2493,7 +2402,9 @@ public class HRegion implements HeapSize
@SuppressWarnings("unchecked")
private void doBatchMutate(Mutation mutation, Integer lid) throws IOException,
org.apache.hadoop.hbase.exceptions.DoNotRetryIOException {
- Pair<Mutation, Integer>[] mutateWithLocks = new Pair[] { new Pair<Mutation, Integer>(mutation, lid) };
+ Pair<Mutation, Integer>[] mutateWithLocks = new Pair[] {
+ new Pair<Mutation, Integer>(mutation, lid)
+ };
OperationStatus[] batchMutate = this.batchMutate(mutateWithLocks);
if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) {
throw new FailedSanityCheckException(batchMutate[0].getExceptionMsg());
@@ -2520,12 +2431,12 @@ public class HRegion implements HeapSize
// This should be "fast" since we don't rewrite store files but instead
// back up the store files by creating a reference
Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
- Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(desc, rootDir,
- regionInfo.getEncodedName());
+ Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing region-info for snapshot.");
- checkRegioninfoOnFilesystem(snapshotRegionDir);
+ HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
+ getFilesystem(), snapshotDir, getRegionInfo());
// 2. iterate through all the stores in the region
LOG.debug("Creating references for hfiles");
@@ -2537,8 +2448,8 @@ public class HRegion implements HeapSize
// files/batch, far more than the number of store files under a single column family.
for (Store store : stores.values()) {
// 2.1. build the snapshot reference directory for the store
- Path dstStoreDir = TakeSnapshotUtils.getStoreSnapshotDirectory(snapshotRegionDir,
- Bytes.toString(store.getFamily().getName()));
+ Path dstStoreDir = TakeSnapshotUtils.getStoreSnapshotDirectory(
+ snapshotRegionFs.getRegionDir(), Bytes.toString(store.getFamily().getName()));
List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
if (LOG.isDebugEnabled()) {
LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
@@ -2557,7 +2468,7 @@ public class HRegion implements HeapSize
// open and read the files as well).
LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
Path referenceFile = new Path(dstStoreDir, file.getName());
- boolean success = fs.createNewFile(referenceFile);
+ boolean success = fs.getFileSystem().createNewFile(referenceFile);
if (!success) {
throw new IOException("Failed to create reference file:" + referenceFile);
}
@@ -2871,15 +2782,16 @@ public class HRegion implements HeapSize
}
long seqid = minSeqIdForTheRegion;
+ FileSystem fs = this.fs.getFileSystem();
NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regiondir);
if (files == null || files.isEmpty()) return seqid;
for (Path edits: files) {
- if (edits == null || !this.fs.exists(edits)) {
+ if (edits == null || !fs.exists(edits)) {
LOG.warn("Null or non-existent edits file: " + edits);
continue;
}
- if (isZeroLengthThenDelete(this.fs, edits)) continue;
+ if (isZeroLengthThenDelete(fs, edits)) continue;
long maxSeqId = Long.MAX_VALUE;
String fileName = edits.getName();
@@ -2917,7 +2829,7 @@ public class HRegion implements HeapSize
// The edits size added into rsAccounting during this replaying will not
// be required any more. So just clear it.
if (this.rsAccounting != null) {
- this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName());
+ this.rsAccounting.clearRegionReplayEditsSize(this.getRegionName());
}
}
if (seqid > minSeqIdForTheRegion) {
@@ -2926,7 +2838,7 @@ public class HRegion implements HeapSize
}
// Now delete the content of recovered edits. We're done w/ them.
for (Path file: files) {
- if (!this.fs.delete(file, false)) {
+ if (!fs.delete(file, false)) {
LOG.error("Failed delete of " + file);
} else {
LOG.debug("Deleted recovered.edits file=" + file);
@@ -2950,11 +2862,12 @@ public class HRegion implements HeapSize
String msg = "Replaying edits from " + edits;
LOG.info(msg);
MonitoredTask status = TaskMonitor.get().createStatus(msg);
+ FileSystem fs = this.fs.getFileSystem();
status.setStatus("Opening logs");
HLog.Reader reader = null;
try {
- reader = HLogFactory.createReader(this.fs, edits, conf);
+ reader = HLogFactory.createReader(fs, edits, conf);
long currentEditSeqId = -1;
long firstSeqIdInLog = -1;
long skippedEdits = 0;
@@ -3019,7 +2932,8 @@ public class HRegion implements HeapSize
// Check this edit is for me. Also, guard against writing the special
// METACOLUMN info such as HBASE::CACHEFLUSH entries
if (kv.matchingFamily(HLog.METAFAMILY) ||
- !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
+ !Bytes.equals(key.getEncodedRegionName(),
+ this.getRegionInfo().getEncodedNameAsBytes())) {
skippedEdits++;
continue;
}
@@ -3102,7 +3016,7 @@ public class HRegion implements HeapSize
protected boolean restoreEdit(final Store s, final KeyValue kv) {
long kvSize = s.add(kv);
if (this.rsAccounting != null) {
- rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize);
+ rsAccounting.addAndGetRegionReplayEditsSize(this.getRegionName(), kvSize);
}
return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
}
@@ -3124,7 +3038,7 @@ public class HRegion implements HeapSize
protected HStore instantiateHStore(Path tableDir, HColumnDescriptor c)
throws IOException {
- return new HStore(tableDir, this, c, this.fs, this.conf);
+ return new HStore(tableDir, this, c, this.getFilesystem(), this.conf);
}
/**
@@ -3172,11 +3086,11 @@ public class HRegion implements HeapSize
/** Make sure this is a valid row for the HRegion */
void checkRow(final byte [] row, String op) throws IOException {
- if(!rowIsInRange(regionInfo, row)) {
+ if (!rowIsInRange(getRegionInfo(), row)) {
throw new WrongRegionException("Requested row out of range for " +
op + " on HRegion " + this + ", startKey='" +
- Bytes.toStringBinary(regionInfo.getStartKey()) + "', getEndKey()='" +
- Bytes.toStringBinary(regionInfo.getEndKey()) + "', row='" +
+ Bytes.toStringBinary(getStartKey()) + "', getEndKey()='" +
+ Bytes.toStringBinary(getEndKey()) + "', row='" +
Bytes.toStringBinary(row) + "'");
}
}
@@ -3463,12 +3377,12 @@ public class HRegion implements HeapSize
@Override
public String toString() {
- return this.regionInfo.getRegionNameAsString();
+ return this.getRegionNameAsString();
}
/** @return Path of region base directory */
public Path getTableDir() {
- return this.tableDir;
+ return this.fs.getTableDir();
}
/**
@@ -3496,7 +3410,7 @@ public class HRegion implements HeapSize
private HRegion region;
public HRegionInfo getRegionInfo() {
- return regionInfo;
+ return region.getRegionInfo();
}
RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region)
@@ -3829,9 +3743,10 @@ public class HRegion implements HeapSize
}
protected boolean nextRow(byte [] currentRow, int offset, short length) throws IOException {
- assert this.joinedContinuationRow == null : "Trying to go to next row during joinedHeap read.";
+ assert this.joinedContinuationRow == null: "Trying to go to next row during joinedHeap read.";
KeyValue next;
- while ((next = this.storeHeap.peek()) != null && next.matchingRow(currentRow, offset, length)) {
+ while ((next = this.storeHeap.peek()) != null &&
+ next.matchingRow(currentRow, offset, length)) {
this.storeHeap.next(MOCKED_LIST);
}
resetFilters();
@@ -3908,7 +3823,7 @@ public class HRegion implements HeapSize
* @param conf is global configuration settings.
* @param regionInfo - HRegionInfo that describes the region
* is new), then read them from the supplied path.
- * @param htd
+ * @param htd the table descriptor
* @param rsServices
* @return the new instance
*/
@@ -3957,7 +3872,8 @@ public class HRegion implements HeapSize
}
/**
- * This will do the necessary cleanup a call to {@link #createHRegion(HRegionInfo, Path, Configuration, HTableDescriptor)}
+ * This will do the necessary cleanup a call to
+ * {@link #createHRegion(HRegionInfo, Path, Configuration, HTableDescriptor)}
* requires. This method will close the region and then close its
* associated {@link HLog} file. You use it if you call the other createHRegion,
* the one that takes an {@link HLog} instance but don't be surprised by the
@@ -4024,16 +3940,12 @@ public class HRegion implements HeapSize
+ " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
" Table name == " + info.getTableNameAsString());
- Path tableDir =
- HTableDescriptor.getTableDir(rootDir, info.getTableName());
- Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
+ Path tableDir = HTableDescriptor.getTableDir(rootDir, info.getTableName());
FileSystem fs = FileSystem.get(conf);
- fs.mkdirs(regionDir);
- // Write HRI to a file in case we need to recover .META.
- writeRegioninfoOnFilesystem(info, regionDir, fs, conf);
+ HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, info);
HLog effectiveHLog = hlog;
if (hlog == null && !ignoreHLog) {
- effectiveHLog = HLogFactory.createHLog(fs, regionDir,
+ effectiveHLog = HLogFactory.createHLog(fs, rfs.getRegionDir(),
HConstants.HREGION_LOGDIR_NAME, conf);
}
HRegion region = HRegion.newHRegion(tableDir,
@@ -4052,6 +3964,7 @@ public class HRegion implements HeapSize
return createHRegion(info, rootDir, conf, hTableDescriptor, hlog, true);
}
+
/**
* Open a Region.
* @param info Info for region to be opened.
@@ -4074,7 +3987,7 @@ public class HRegion implements HeapSize
/**
* Open a Region.
* @param info Info for region to be opened
- * @param htd
+ * @param htd the table descriptor
* @param wal HLog for region to use. This method will call
* HLog#setSequenceNumber(long) passing the result of the call to
* HRegion#getMinSequenceId() to ensure the log id is properly kept
@@ -4241,11 +4154,17 @@ public class HRegion implements HeapSize
*/
HRegion createDaughterRegion(final HRegionInfo hri, final Path daughterTmpDir)
throws IOException {
+ FileSystem fs = this.fs.getFileSystem();
HRegion r = HRegion.newHRegion(this.getTableDir(), this.getLog(), fs,
this.getBaseConf(), hri, this.getTableDesc(), rsServices);
r.readRequestsCount.set(this.getReadRequestsCount() / 2);
r.writeRequestsCount.set(this.getWriteRequestsCount() / 2);
- moveInitialFilesIntoPlace(fs, daughterTmpDir, r.getRegionDir());
+ // Move the tmp dir in the expected location
+ if (daughterTmpDir != null && fs.exists(daughterTmpDir)) {
+ if (!fs.rename(daughterTmpDir, r.getRegionDir())) {
+ LOG.warn("Unable to rename " + daughterTmpDir + " to " + r.getRegionDir());
+ }
+ }
return r;
}
@@ -4259,6 +4178,7 @@ public class HRegion implements HeapSize
*/
HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
final HRegion region_b, final Path mergedTmpDir) throws IOException {
+ FileSystem fs = this.getFilesystem();
HRegion r = HRegion.newHRegion(this.getTableDir(), this.getLog(), fs,
this.getBaseConf(), mergedRegionInfo, this.getTableDesc(),
this.rsServices);
@@ -4305,26 +4225,15 @@ public class HRegion implements HeapSize
}
/**
- * Deletes all the files for a HRegion
+ * Computes the Path of the HRegion
*
- * @param fs the file system object
- * @param rootdir qualified path of HBase root directory
- * @param info HRegionInfo for region to be deleted
- * @throws IOException
+ * @param tabledir qualified path for table
+ * @param name ENCODED region name
+ * @return Path of HRegion directory
*/
- public static void deleteRegion(FileSystem fs, Path rootdir, HRegionInfo info)
- throws IOException {
- deleteRegion(fs, HRegion.getRegionDir(rootdir, info));
- }
-
- private static void deleteRegion(FileSystem fs, Path regiondir)
- throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("DELETING region " + regiondir.toString());
- }
- if (!fs.delete(regiondir, true)) {
- LOG.warn("Failed delete of " + regiondir);
- }
+ @Deprecated
+ public static Path getRegionDir(final Path tabledir, final String name) {
+ return new Path(tabledir, name);
}
/**
@@ -4334,6 +4243,7 @@ public class HRegion implements HeapSize
* @param info HRegionInfo for the region
* @return qualified path of region directory
*/
+ @Deprecated
public static Path getRegionDir(final Path rootdir, final HRegionInfo info) {
return new Path(
HTableDescriptor.getTableDir(rootdir, info.getTableName()),
@@ -4782,7 +4692,7 @@ public class HRegion implements HeapSize
long txid = 0;
// 7. Append no sync
if (!walEdit.isEmpty()) {
- txid = this.log.appendNoSync(this.regionInfo,
+ txid = this.log.appendNoSync(this.getRegionInfo(),
this.htableDescriptor.getName(), walEdit,
processor.getClusterId(), now, this.htableDescriptor);
}
@@ -5014,7 +4924,7 @@ public class HRegion implements HeapSize
// Using default cluster id, as this can only happen in the orginating
// cluster. A slave cluster receives the final value (not the delta)
// as a Put.
- txid = this.log.appendNoSync(regionInfo,
+ txid = this.log.appendNoSync(this.getRegionInfo(),
this.htableDescriptor.getName(), walEdits,
HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
this.htableDescriptor);
@@ -5158,7 +5068,7 @@ public class HRegion implements HeapSize
// Using default cluster id, as this can only happen in the orginating
// cluster. A slave cluster receives the final value (not the delta)
// as a Put.
- txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
+ txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(),
walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
this.htableDescriptor);
}
@@ -5221,7 +5131,7 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
- 41 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+ 38 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
(10 * Bytes.SIZEOF_LONG) +
Bytes.SIZEOF_BOOLEAN);
@@ -5265,7 +5175,8 @@ public class HRegion implements HeapSize
/**
* Registers a new protocol buffer {@link Service} subclass as a coprocessor endpoint to
* be available for handling
- * {@link HRegion#execService(com.google.protobuf.RpcController, org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall)}} calls.
+ * {@link HRegion#execService(com.google.protobuf.RpcController,
+ * org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall)}} calls.
*
* <p>
* Only a single instance may be registered per region for a given {@link Service} subclass (the
@@ -5300,7 +5211,7 @@ public class HRegion implements HeapSize
/**
* Executes a single protocol buffer coprocessor endpoint {@link Service} method using
* the registered protocol handlers. {@link Service} implementations must be registered via the
- * {@link org.apache.hadoop.hbase.regionserver.HRegion#registerService(com.google.protobuf.Service)}
+ * {@link HRegion#registerService(com.google.protobuf.Service)}
* method before they are available.
*
* @param controller an {@code RpcContoller} implementation to pass to the invoked service
@@ -5431,7 +5342,7 @@ public class HRegion implements HeapSize
*/
public byte[] checkSplit() {
// Can't split META
- if (this.regionInfo.isMetaTable()) {
+ if (this.getRegionInfo().isMetaTable()) {
if (shouldForceSplit()) {
LOG.warn("Cannot split meta region in HBase 0.20 and above");
}
@@ -5502,14 +5413,12 @@ public class HRegion implements HeapSize
public void startRegionOperation()
throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
if (this.closing.get()) {
- throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
- " is closing");
+ throw new NotServingRegionException(getRegionNameAsString() + " is closing");
}
lock(lock.readLock());
if (this.closed.get()) {
lock.readLock().unlock();
- throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
- " is closed");
+ throw new NotServingRegionException(getRegionNameAsString() + " is closed");
}
}
@@ -5533,16 +5442,14 @@ public class HRegion implements HeapSize
private void startBulkRegionOperation(boolean writeLockNeeded)
throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
if (this.closing.get()) {
- throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
- " is closing");
+ throw new NotServingRegionException(getRegionNameAsString() + " is closing");
}
if (writeLockNeeded) lock(lock.writeLock());
else lock(lock.readLock());
if (this.closed.get()) {
if (writeLockNeeded) lock.writeLock().unlock();
else lock.readLock().unlock();
- throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
- " is closed");
+ throw new NotServingRegionException(getRegionNameAsString() + " is closed");
}
}
@@ -5611,7 +5518,7 @@ public class HRegion implements HeapSize
* @throws IOException If anything goes wrong with DFS
*/
private void syncOrDefer(long txid) throws IOException {
- if (this.regionInfo.isMetaRegion() ||
+ if (this.getRegionInfo().isMetaRegion() ||
!this.htableDescriptor.isDeferredLogFlush() || this.deferredLogSyncDisabled) {
this.log.sync(txid);
}
Added: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java?rev=1460611&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java (added)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java Mon Mar 25 11:08:56 2013
@@ -0,0 +1,362 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * View to an on-disk Region.
+ * Provides the set of methods necessary to interact with the on-disk region data.
+ */
+@InterfaceAudience.Private
+public class HRegionFileSystem {
+ public static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
+
+ /** Name of the region info file that resides just under the region directory. */
+ public final static String REGION_INFO_FILE = ".regioninfo";
+
+ /** Temporary subdirectory of the region directory used for merges. */
+ public static final String REGION_MERGES_DIR = ".merges";
+
+ /** Temporary subdirectory of the region directory used for splits. */
+ public static final String REGION_SPLITS_DIR = ".splits";
+
+ /** Temporary subdirectory of the region directory used for compaction output. */
+ private static final String REGION_TEMP_DIR = ".tmp";
+
+ private final HRegionInfo regionInfo;
+ private final Configuration conf;
+ private final Path tableDir;
+ private final FileSystem fs;
+
+ /**
+ * Create a view to the on-disk region
+ * @param conf the {@link Configuration} to use
+ * @param fs {@link FileSystem} that contains the region
+ * @param tableDir {@link Path} to where the table is being stored
+ * @param regionInfo {@link HRegionInfo} for region
+ */
+ HRegionFileSystem(final Configuration conf, final FileSystem fs,
+ final Path tableDir, final HRegionInfo regionInfo) {
+ this.fs = fs;
+ this.conf = conf;
+ this.tableDir = tableDir;
+ this.regionInfo = regionInfo;
+ }
+
+ /** @return the underlying {@link FileSystem} */
+ public FileSystem getFileSystem() {
+ return this.fs;
+ }
+
+ /** @return the {@link HRegionInfo} that describe this on-disk region view */
+ public HRegionInfo getRegionInfo() {
+ return this.regionInfo;
+ }
+
+ /** @return {@link Path} to the region's root directory. */
+ public Path getTableDir() {
+ return this.tableDir;
+ }
+
+ /** @return {@link Path} to the region directory. */
+ public Path getRegionDir() {
+ return new Path(this.tableDir, this.regionInfo.getEncodedName());
+ }
+
+ // ===========================================================================
+ // Temp Helpers
+ // ===========================================================================
+ /** @return {@link Path} to the region's temp directory, used for file creations */
+ public Path getTempDir() {
+ return new Path(getRegionDir(), REGION_TEMP_DIR);
+ }
+
+ /**
+ * Clean up any temp detritus that may have been left around from previous operation attempts.
+ */
+ public void cleanupTempDir() throws IOException {
+ FSUtils.deleteDirectory(fs, getTempDir());
+ }
+
+ // ===========================================================================
+ // Splits Helpers
+ // ===========================================================================
+ /** @return {@link Path} to the temp directory used during split operations */
+ public Path getSplitsDir() {
+ return new Path(getRegionDir(), REGION_SPLITS_DIR);
+ }
+
+ /**
+ * Clean up any split detritus that may have been left around from previous split attempts.
+ */
+ public void cleanupSplitsDir() throws IOException {
+ FSUtils.deleteDirectory(fs, getSplitsDir());
+ }
+
+ // ===========================================================================
+ // Merge Helpers
+ // ===========================================================================
+ /** @return {@link Path} to the temp directory used during merge operations */
+ public Path getMergesDir() {
+ return new Path(getRegionDir(), REGION_MERGES_DIR);
+ }
+
+ /**
+ * Clean up any merge detritus that may have been left around from previous merge attempts.
+ */
+ public void cleanupMergesDir() throws IOException {
+ FSUtils.deleteDirectory(fs, getMergesDir());
+ }
+
+ // ===========================================================================
+ // Create/Open/Delete Helpers
+ // ===========================================================================
+ /**
+ * @param hri
+ * @return Content of the file we write out to the filesystem under a region
+ * @throws IOException
+ */
+ private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
+ return hri.toDelimitedByteArray();
+ }
+
+ /**
+ * Create a {@link HRegionInfo} from the serialized version on-disk.
+ * @param fs {@link FileSystem} that contains the Region Info file
+ * @param regionDir {@link Path} to the Region Directory that contains the Info file
+ * @return An {@link HRegionInfo} instance gotten from the Region Info file.
+ * @throws IOException if an error occurred during file open/read operation.
+ */
+ public static HRegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
+ throws IOException {
+ FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
+ try {
+ return HRegionInfo.parseFrom(in);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Write the .regioninfo file on-disk.
+ */
+ private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
+ final Path regionInfoFile, final byte[] content) throws IOException {
+ // First check to get the permissions
+ FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
+ // Write the RegionInfo file content
+ FSDataOutputStream out = FSUtils.create(fs, regionInfoFile, perms);
+ try {
+ out.write(content);
+ } finally {
+ out.close();
+ }
+ }
+
+ /**
+ * Write out an info file under the stored region directory. Useful recovering mangled regions.
+ * If the regionInfo already exists on-disk, then we fast exit.
+ */
+ void checkRegionInfoOnFilesystem() throws IOException {
+ // Compose the content of the file so we can compare to length in filesystem. If not same,
+ // rewrite it (it may have been written in the old format using Writables instead of pb). The
+ // pb version is much shorter -- we write now w/o the toString version -- so checking length
+ // only should be sufficient. I don't want to read the file every time to check if it pb
+ // serialized.
+ byte[] content = getRegionInfoFileContent(regionInfo);
+ try {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+
+ FileStatus status = fs.getFileStatus(regionInfoFile);
+ if (status != null && status.getLen() == content.length) {
+ // Then assume the content good and move on.
+ // NOTE: that the length is not sufficient to define the the content matches.
+ return;
+ }
+
+ LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
+ if (!fs.delete(regionInfoFile, false)) {
+ throw new IOException("Unable to remove existing " + regionInfoFile);
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfo.getEncodedName());
+ }
+
+ // Write HRI to a file in case we need to recover .META.
+ writeRegionInfoOnFilesystem(content, true);
+ }
+
+ /**
+ * Write out an info file under the region directory. Useful recovering mangled regions.
+ * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
+ */
+ private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
+ byte[] content = getRegionInfoFileContent(regionInfo);
+ writeRegionInfoOnFilesystem(content, useTempDir);
+ }
+
+ /**
+ * Write out an info file under the region directory. Useful recovering mangled regions.
+ * @param regionInfoContent serialized version of the {@link HRegionInfo}
+ * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
+ */
+ private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent,
+ final boolean useTempDir) throws IOException {
+ Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
+ if (useTempDir) {
+ // Create in tmpDir and then move into place in case we crash after
+ // create but before close. If we don't successfully close the file,
+ // subsequent region reopens will fail the below because create is
+ // registered in NN.
+
+ // And then create the file
+ Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
+
+ // If datanode crashes or if the RS goes down just before the close is called while trying to
+ // close the created regioninfo file in the .tmp directory then on next
+ // creation we will be getting AlreadyCreatedException.
+ // Hence delete and create the file if exists.
+ if (FSUtils.isExists(fs, tmpPath)) {
+ FSUtils.delete(fs, tmpPath, true);
+ }
+
+ // Write HRI to a file in case we need to recover .META.
+ writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
+
+ // Move the created file to the original path
+ if (!fs.rename(tmpPath, regionInfoFile)) {
+ throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
+ }
+ } else {
+ // Write HRI to a file in case we need to recover .META.
+ writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
+ }
+ }
+
+ /**
+ * Create a new Region on file-system.
+ * @param conf the {@link Configuration} to use
+ * @param fs {@link FileSystem} from which to add the region
+ * @param tableDir {@link Path} to where the table is being stored
+ * @param regionInfo {@link HRegionInfo} for region to be added
+ * @throws IOException if the region creation fails due to a FileSystem exception.
+ */
+ public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
+ final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
+ Path regionDir = regionFs.getRegionDir();
+
+ if (fs.exists(regionDir)) {
+ LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
+ throw new IOException("The specified region already exists on disk: " + regionDir);
+ }
+
+ // Create the region directory
+ if (!fs.mkdirs(regionFs.getRegionDir())) {
+ LOG.warn("Unable to create the region directory: " + regionDir);
+ throw new IOException("Unable to create region directory: " + regionDir);
+ }
+
+ // Write HRI to a file in case we need to recover .META.
+ regionFs.writeRegionInfoOnFilesystem(false);
+ return regionFs;
+ }
+
+ /**
+ * Open Region from file-system.
+ * @param conf the {@link Configuration} to use
+ * @param fs {@link FileSystem} from which to add the region
+ * @param tableDir {@link Path} to where the table is being stored
+ * @param regionInfo {@link HRegionInfo} for region to be added
+ * @throws IOException if the region creation fails due to a FileSystem exception.
+ */
+ public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
+ final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
+ Path regionDir = regionFs.getRegionDir();
+
+ if (!fs.exists(regionDir)) {
+ LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
+ throw new IOException("The specified region do not exists on disk: " + regionDir);
+ }
+
+ // Cleanup temporary directories
+ regionFs.cleanupTempDir();
+ regionFs.cleanupSplitsDir();
+ regionFs.cleanupMergesDir();
+ // if it doesn't exists, Write HRI to a file, in case we need to recover .META.
+ regionFs.checkRegionInfoOnFilesystem();
+ return regionFs;
+ }
+
+ /**
+ * Remove the region from the table directory, archiving the region's hfiles.
+ * @param conf the {@link Configuration} to use
+ * @param fs {@link FileSystem} from which to remove the region
+ * @param tableDir {@link Path} to where the table is being stored
+ * @param regionInfo {@link HRegionInfo} for region to be deleted
+ * @throws IOException if the request cannot be completed
+ */
+ public static void deleteRegionFromFileSystem(final Configuration conf,
+ final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
+ Path regionDir = regionFs.getRegionDir();
+
+ if (!fs.exists(regionDir)) {
+ LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
+ return;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("DELETING region " + regionDir);
+ }
+
+ // Archive region
+ Path rootDir = FSUtils.getRootDir(conf);
+ HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
+
+ // Delete empty region dir
+ if (!fs.delete(regionDir, true)) {
+ LOG.warn("Failed delete of " + regionDir);
+ }
+ }
+}
\ No newline at end of file
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java Mon Mar 25 11:08:56 2013
@@ -83,7 +83,6 @@ import com.google.common.util.concurrent
@InterfaceAudience.Private
public class SplitTransaction {
private static final Log LOG = LogFactory.getLog(SplitTransaction.class);
- private static final String SPLITDIR = ".splits";
/*
* Region to split
@@ -544,7 +543,7 @@ public class SplitTransaction {
}
private static Path getSplitDir(final HRegion r) {
- return new Path(r.getRegionDir(), SPLITDIR);
+ return new Path(r.getRegionDir(), HRegionFileSystem.REGION_SPLITS_DIR);
}
/**
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java Mon Mar 25 11:08:56 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.errorhand
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -173,7 +174,7 @@ public class RestoreSnapshotHelper {
for (String regionName: snapshotRegionNames) {
LOG.info("region to add: " + regionName);
Path regionDir = new Path(snapshotDir, regionName);
- regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir));
+ regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
}
// Create new regions cloning from the snapshot
@@ -562,7 +563,7 @@ public class RestoreSnapshotHelper {
List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
for (FileStatus regionDir: regionDirs) {
- HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir.getPath());
+ HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
regions.add(hri);
}
LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString());
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Mon Mar 25 11:08:56 2013
@@ -87,6 +87,7 @@ import org.apache.hadoop.hbase.io.hfile.
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.security.User;
@@ -723,7 +724,9 @@ public class HBaseFsck extends Configure
// already loaded data
return;
}
- HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(FileSystem.get(getConf()), regionDir);
+
+ FileSystem fs = FileSystem.get(getConf());
+ HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
LOG.debug("HRegionInfo read: " + hri.toString());
hbi.hdfsEntry.hri = hri;
}
@@ -1844,7 +1847,7 @@ public class HBaseFsck extends Configure
Path src = cf.getPath();
Path dst = new Path(targetRegionDir, src.getName());
- if (src.getName().equals(HRegion.REGIONINFO_FILE)) {
+ if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
// do not copy the old .regioninfo file.
continue;
}
@@ -3137,7 +3140,7 @@ public class HBaseFsck extends Configure
he.hdfsRegionDir = regionDir.getPath();
he.hdfsRegionDirModTime = regionDir.getModificationTime();
- Path regioninfoFile = new Path(he.hdfsRegionDir, HRegion.REGIONINFO_FILE);
+ Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
// we add to orphan list when we attempt to read .regioninfo
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java Mon Mar 25 11:08:56 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.client.HB
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -424,11 +425,11 @@ public class TestHFileArchiving {
// remove all the non-storefile named files for the region
for (int i = 0; i < storeFiles.size(); i++) {
String file = storeFiles.get(i);
- if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
+ if (file.contains(HRegionFileSystem.REGION_INFO_FILE) || file.contains("hlog")) {
storeFiles.remove(i--);
}
}
- storeFiles.remove(HRegion.REGIONINFO_FILE);
+ storeFiles.remove(HRegionFileSystem.REGION_INFO_FILE);
return storeFiles;
}
}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Mon Mar 25 11:08:56 2013
@@ -3496,10 +3496,10 @@ public class TestHRegion extends HBaseTe
FileSystem fs = region.getFilesystem();
HRegion.closeHRegion(region);
- Path regionInfoFile = new Path(regionDir, HRegion.REGIONINFO_FILE);
+ Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
// Verify that the .regioninfo file is present
- assertTrue(HRegion.REGIONINFO_FILE + " should be present in the region dir",
+ assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
fs.exists(regionInfoFile));
// Try to open the region
@@ -3508,12 +3508,12 @@ public class TestHRegion extends HBaseTe
HRegion.closeHRegion(region);
// Verify that the .regioninfo file is still there
- assertTrue(HRegion.REGIONINFO_FILE + " should be present in the region dir",
+ assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
fs.exists(regionInfoFile));
// Remove the .regioninfo file and verify is recreated on region open
fs.delete(regionInfoFile);
- assertFalse(HRegion.REGIONINFO_FILE + " should be removed from the region dir",
+ assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
fs.exists(regionInfoFile));
region = HRegion.openHRegion(rootDir, hri, htd, null, conf);
@@ -3521,8 +3521,8 @@ public class TestHRegion extends HBaseTe
HRegion.closeHRegion(region);
// Verify that the .regioninfo file is still there
- assertTrue(HRegion.REGIONINFO_FILE + " should be present in the region dir",
- fs.exists(new Path(regionDir, HRegion.REGIONINFO_FILE)));
+ assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
+ fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
}
/**
Added: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java?rev=1460611&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java (added)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java Mon Mar 25 11:08:56 2013
@@ -0,0 +1,74 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import org.junit.Test;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import junit.framework.TestCase;
+
+@Category(SmallTests.class)
+public class TestHRegionFileSystem {
+ private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Test
+ public void testOnDiskRegionCreation() throws IOException {
+ Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testOnDiskRegionCreation");
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Configuration conf = TEST_UTIL.getConfiguration();
+
+ // Create a Region
+ HRegionInfo hri = new HRegionInfo(Bytes.toBytes("TestTable"));
+ HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri);
+
+ // Verify if the region is on disk
+ Path regionDir = regionFs.getRegionDir();
+ assertTrue("The region folder should be created", fs.exists(regionDir));
+
+ // Verify the .regioninfo
+ HRegionInfo hriVerify = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+ assertEquals(hri, hriVerify);
+
+ // Open the region
+ regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, rootDir, hri);
+ assertEquals(regionDir, regionFs.getRegionDir());
+
+ // Delete the region
+ HRegionFileSystem.deleteRegionFromFileSystem(conf, fs, rootDir, hri);
+ assertFalse("The region folder should be removed", fs.exists(regionDir));
+
+ fs.delete(rootDir, true);
+ }
+}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java Mon Mar 25 11:08:56 2013
@@ -61,19 +61,20 @@ public class TestHRegionInfo {
long modtime = getModTime(r);
HRegion.closeHRegion(r);
Thread.sleep(1001);
- r = HRegion.createHRegion(hri, basedir, htu.getConfiguration(), HTableDescriptor.META_TABLEDESC);
+ r = HRegion.openHRegion(basedir, hri, HTableDescriptor.META_TABLEDESC,
+ null, htu.getConfiguration());
// Ensure the file is not written for a second time.
long modtime2 = getModTime(r);
assertEquals(modtime, modtime2);
// Now load the file.
- HRegionInfo deserializedHri =
- HRegion.loadDotRegionInfoFileContent(FileSystem.get(htu.getConfiguration()), r.getRegionDir());
+ HRegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent(
+ FileSystem.get(htu.getConfiguration()), r.getRegionDir());
assertTrue(hri.equals(deserializedHri));
}
long getModTime(final HRegion r) throws IOException {
FileStatus [] statuses =
- r.getFilesystem().listStatus(new Path(r.getRegionDir(), HRegion.REGIONINFO_FILE));
+ r.getFilesystem().listStatus(new Path(r.getRegionDir(), HRegionFileSystem.REGION_INFO_FILE));
assertTrue(statuses != null && statuses.length == 1);
return statuses[0].getModificationTime();
}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Mon Mar 25 11:08:56 2013
@@ -255,7 +255,6 @@ public class TestWALReplay {
HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
Path basedir = new Path(hbaseRootDir, tableNameStr);
deleteDir(basedir);
- fs.mkdirs(new Path(basedir, hri.getEncodedName()));
HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
HRegion region2 = HRegion.createHRegion(hri,
@@ -683,7 +682,7 @@ public class TestWALReplay {
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
final Path basedir = new Path(hbaseRootDir, tableNameStr);
deleteDir(basedir);
- fs.mkdirs(new Path(basedir, hri.getEncodedName()));
+
final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
HRegion region2 = HRegion.createHRegion(hri,
hbaseRootDir, this.conf, htd);
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java Mon Mar 25 11:08:56 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -196,8 +197,7 @@ public class SnapshotTestingUtils {
for (HRegionInfo info : regions) {
String regionName = info.getEncodedName();
Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs,
- regionDir);
+ HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
assertEquals(info, snapshotRegionInfo);
// check to make sure we have the family
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java Mon Mar 25 11:08:56 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.master.sn
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -292,7 +293,7 @@ public class TestFlushSnapshotFromClient
for (HRegionInfo info : regions) {
String regionName = info.getEncodedName();
Path regionDir = new Path(snapshotDir, regionName);
- HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
+ HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
assertEquals(info, snapshotRegionInfo);
// check to make sure we have the family
Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java?rev=1460611&r1=1460610&r2=1460611&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java Mon Mar 25 11:08:56 2013
@@ -69,7 +69,7 @@ import org.apache.hadoop.hbase.master.As
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
@@ -277,7 +277,7 @@ public class TestHBaseFsck {
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
- Path hriPath = new Path(p, HRegion.REGIONINFO_FILE);
+ Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
fs.delete(hriPath, true);
}