You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2014/04/18 19:15:59 UTC
svn commit: r1588517 - in
/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver:
HRegionFileSystem.java HStore.java StoreFile.java
Author: mbertozzi
Date: Fri Apr 18 17:15:58 2014
New Revision: 1588517
URL: http://svn.apache.org/r1588517
Log:
HBASE-11011 Avoid extra getFileStatus() calls on Region startup
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java?rev=1588517&r1=1588516&r2=1588517&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java Fri Apr 18 17:15:58 2014
@@ -197,6 +197,32 @@ public class HRegionFileSystem {
}
/**
+ * Return Qualified Path of the specified family/file
+ *
+ * @param familyName Column Family Name
+ * @param fileName File Name
+ * @return The qualified Path for the specified family/file
+ */
+ Path getStoreFilePath(final String familyName, final String fileName) {
+ Path familyDir = getStoreDir(familyName);
+ return new Path(familyDir, fileName).makeQualified(this.fs);
+ }
+
+ /**
+ * Return the store file information of the specified family/file.
+ *
+ * @param familyName Column Family Name
+ * @param fileName File Name
+ * @return The {@link StoreFileInfo} for the specified family/file
+ */
+ StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+ throws IOException {
+ Path familyDir = getStoreDir(familyName);
+ FileStatus status = fs.getFileStatus(new Path(familyDir, fileName));
+ return new StoreFileInfo(this.conf, this.fs, status);
+ }
+
+ /**
* Returns true if the specified family has reference files
* @param familyName Column Family Name
* @return true if family contains reference files
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1588517&r1=1588516&r2=1588517&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java Fri Apr 18 17:15:58 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.io.FileNotFoundException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.security.Key;
@@ -501,7 +502,7 @@ public class HStore implements Store {
completionService.submit(new Callable<StoreFile>() {
@Override
public StoreFile call() throws IOException {
- StoreFile storeFile = createStoreFileAndReader(storeFileInfo.getPath());
+ StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
return storeFile;
}
});
@@ -537,7 +538,7 @@ public class HStore implements Store {
for (StoreFile file : results) {
try {
if (file != null) file.closeReader(true);
- } catch (IOException e) {
+ } catch (IOException e) {
LOG.warn(e.getMessage());
}
}
@@ -549,6 +550,11 @@ public class HStore implements Store {
private StoreFile createStoreFileAndReader(final Path p) throws IOException {
StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);
+ return createStoreFileAndReader(info);
+ }
+
+ private StoreFile createStoreFileAndReader(final StoreFileInfo info)
+ throws IOException {
info.setRegionCoprocessorHost(this.region.getCoprocessorHost());
StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf,
this.family.getBloomFilterType());
@@ -1077,7 +1083,7 @@ public class HStore implements Store {
// TODO: get rid of this!
if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
LOG.warn("hbase.hstore.compaction.complete is set to false");
- sfs = new ArrayList<StoreFile>();
+ sfs = new ArrayList<StoreFile>(newFiles.size());
for (Path newFile : newFiles) {
// Create storefile around what we wrote with a reader on it.
StoreFile sf = createStoreFileAndReader(newFile);
@@ -1101,7 +1107,7 @@ public class HStore implements Store {
private List<StoreFile> moveCompatedFilesIntoPlace(
CompactionRequest cr, List<Path> newFiles) throws IOException {
- List<StoreFile> sfs = new ArrayList<StoreFile>();
+ List<StoreFile> sfs = new ArrayList<StoreFile>(newFiles.size());
for (Path newFile : newFiles) {
assert newFile != null;
StoreFile sf = moveFileIntoPlace(newFile);
@@ -1130,7 +1136,7 @@ public class HStore implements Store {
private void writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
Collection<StoreFile> newFiles) throws IOException {
if (region.getLog() == null) return;
- List<Path> inputPaths = new ArrayList<Path>();
+ List<Path> inputPaths = new ArrayList<Path>(filesCompacted.size());
for (StoreFile f : filesCompacted) {
inputPaths.add(f.getPath());
}
@@ -1202,7 +1208,7 @@ public class HStore implements Store {
/**
* Call to complete a compaction. Its for the case where we find in the WAL a compaction
* that was not finished. We could find one recovering a WAL after a regionserver crash.
- * See HBASE-2331.
+ * See HBASE-2231.
* @param compaction
*/
@Override
@@ -1210,43 +1216,37 @@ public class HStore implements Store {
throws IOException {
LOG.debug("Completing compaction from the WAL marker");
List<String> compactionInputs = compaction.getCompactionInputList();
- List<String> compactionOutputs = compaction.getCompactionOutputList();
- List<StoreFile> outputStoreFiles = new ArrayList<StoreFile>(compactionOutputs.size());
- for (String compactionOutput : compactionOutputs) {
- //we should have this store file already
- boolean found = false;
- Path outputPath = new Path(fs.getStoreDir(family.getNameAsString()), compactionOutput);
- outputPath = outputPath.makeQualified(fs.getFileSystem());
- for (StoreFile sf : this.getStorefiles()) {
- if (sf.getPath().makeQualified(sf.getPath().getFileSystem(conf)).equals(outputPath)) {
- found = true;
- break;
- }
- }
- if (!found) {
- if (getFileSystem().exists(outputPath)) {
- outputStoreFiles.add(createStoreFileAndReader(outputPath));
- }
- }
- }
+ // The Compaction Marker is written after the compaction is completed,
+ // and the files moved into the region/family folder.
+ //
+ // If we crash after the entry is written, we may not have removed the
+ // input files, but the output file is present.
+ // (The unremoved input files will be removed by this function)
+ //
+ // If we scan the directory and the file is not present, it can mean that:
+ // - The file was manually removed by the user
+ // - The file was removed as consequence of subsequent compaction
+ // so, we can't do anything with the "compaction output list" because those
+ // files have already been loaded when opening the region (by virtue of
+ // being in the store's folder) or they may be missing due to a compaction.
+ String familyName = this.getColumnFamilyName();
List<Path> inputPaths = new ArrayList<Path>(compactionInputs.size());
for (String compactionInput : compactionInputs) {
- Path inputPath = new Path(fs.getStoreDir(family.getNameAsString()), compactionInput);
- inputPath = inputPath.makeQualified(fs.getFileSystem());
+ Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
inputPaths.add(inputPath);
}
//some of the input files might already be deleted
List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size());
for (StoreFile sf : this.getStorefiles()) {
- if (inputPaths.contains(sf.getPath().makeQualified(fs.getFileSystem()))) {
+ if (inputPaths.contains(sf.getQualifiedPath())) {
inputStoreFiles.add(sf);
}
}
- this.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
+ this.replaceStoreFiles(inputStoreFiles, Collections.EMPTY_LIST);
this.completeCompaction(inputStoreFiles);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1588517&r1=1588516&r2=1588517&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Fri Apr 18 17:15:58 2014
@@ -234,6 +234,13 @@ public class StoreFile {
}
/**
+ * @return Returns the qualified path of this StoreFile
+ */
+ public Path getQualifiedPath() {
+ return this.fileInfo.getPath().makeQualified(fs);
+ }
+
+ /**
* @return True if this is a StoreFile Reference; call after {@link #open()}
* else may get wrong answer.
*/