You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/05/28 23:01:58 UTC
svn commit: r661082 - in /hadoop/hbase/branches/0.1: CHANGES.txt
src/java/org/apache/hadoop/hbase/HStore.java
src/java/org/apache/hadoop/hbase/HStoreFile.java
Author: stack
Date: Wed May 28 14:01:58 2008
New Revision: 661082
URL: http://svn.apache.org/viewvc?rev=661082&view=rev
Log:
HBASE-648 If mapfile index is empty, run repair
Modified:
hadoop/hbase/branches/0.1/CHANGES.txt
hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java
hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java
Modified: hadoop/hbase/branches/0.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/CHANGES.txt?rev=661082&r1=661081&r2=661082&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.1/CHANGES.txt Wed May 28 14:01:58 2008
@@ -7,6 +7,7 @@
HBASE-642 Splitting log in a hostile environment -- bad hdfs -- we drop
write-ahead-log edits
HBASE-646 EOFException opening HStoreFile info file (spin on HBASE-645 and 550)
+ HBASE-648 If mapfile index is empty, run repair
Release 0.1.2 - 05/13/2008
Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java?rev=661082&r1=661081&r2=661082&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java Wed May 28 14:01:58 2008
@@ -46,6 +46,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HStoreFile.HbaseMapFile;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.TextSequence;
@@ -855,24 +856,8 @@
// Finally, start up all the map readers! (There could be more than one
// since we haven't compacted yet.)
for(Map.Entry<Long, HStoreFile> e: this.storefiles.entrySet()) {
- MapFile.Reader r = null;
- try {
- r = e.getValue().getReader(this.fs, this.bloomFilter);
- } catch (EOFException eofe) {
- LOG.warn("Failed open of reader " + e.toString() + "; attempting fix",
- eofe);
- try {
- // Try fixing this file.. if we can.
- MapFile.fix(this.fs, e.getValue().getMapFilePath(),
- HStoreFile.HbaseMapFile.KEY_CLASS,
- HStoreFile.HbaseMapFile.VALUE_CLASS, false, this.conf);
- } catch (Exception fixe) {
- LOG.warn("Failed fix of " + e.toString() +
- "...continuing; Probable DATA LOSS!!!", fixe);
- continue;
- }
- }
- this.readers.put(e.getKey(), r);
+ this.readers.put(e.getKey(),
+ e.getValue().getReader(this.fs, this.bloomFilter));
}
}
@@ -978,7 +963,8 @@
/*
* Creates a series of HStoreFiles loaded from the given directory.
* There must be a matching 'mapdir' and 'loginfo' pair of files.
- * If only one exists, we'll delete it.
+ * If only one exists, we'll delete it. Does other consistency tests
+ * checking files are not zero, etc.
*
* @param infodir qualified path for info file directory
* @param mapdir qualified path for map file directory
@@ -1034,6 +1020,23 @@
"Deleting. Continuing...Probable DATA LOSS!!! See HBASE-646.");
continue;
}
+ if (isEmptyIndexFile(mapfile)) {
+ try {
+ // Try fixing this file.. if we can. Use the hbase version of fix.
+ // Need to remove the old index file first else fix won't go ahead.
+ this.fs.delete(new Path(mapfile, MapFile.INDEX_FILE_NAME));
+ long count = MapFile.fix(this.fs, mapfile, HbaseMapFile.KEY_CLASS,
+ HbaseMapFile.VALUE_CLASS, false, this.conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Fixed index on " + mapfile.toString() + "; had " +
+ count + " entries");
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed fix of " + mapfile.toString() +
+ "...continuing; Probable DATA LOSS!!!", e);
+ continue;
+ }
+ }
// TODO: Confirm referent exists.
@@ -1070,9 +1073,32 @@
throws IOException {
// Mapfiles are made of 'data' and 'index' files. Confirm 'data' is
// non-null if it exists (may not have been written to yet).
- Path dataFile = new Path(mapfile, "data");
- return this.fs.exists(dataFile) &&
- this.fs.getFileStatus(dataFile).getLen() == 0;
+ return isEmptyFile(new Path(mapfile, MapFile.DATA_FILE_NAME));
+ }
+
+ /*
+ * @param mapfile
+ * @return True if the passed mapfile has a zero-length index component (its
+ * broken).
+ * @throws IOException
+ */
+ private boolean isEmptyIndexFile(final Path mapfile)
+ throws IOException {
+ // Mapfiles are made of 'data' and 'index' files. Confirm 'data' is
+ // non-null if it exists (may not have been written to yet).
+ return isEmptyFile(new Path(mapfile, MapFile.INDEX_FILE_NAME));
+ }
+
+ /*
+ * @param mapfile
+ * @return True if the passed mapfile has a zero-length index component (its
+ * broken).
+ * @throws IOException
+ */
+ private boolean isEmptyFile(final Path f)
+ throws IOException {
+ return this.fs.exists(f) &&
+ this.fs.getFileStatus(f).getLen() == 0;
}
//////////////////////////////////////////////////////////////////////////////
Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java?rev=661082&r1=661081&r2=661082&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStoreFile.java Wed May 28 14:01:58 2008
@@ -21,10 +21,10 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
@@ -538,8 +538,9 @@
* Hbase customizations of MapFile.
*/
static class HbaseMapFile extends MapFile {
- static final Class KEY_CLASS = HStoreKey.class;
- static final Class VALUE_CLASS = ImmutableBytesWritable.class;
+ static final Class<? extends Writable> KEY_CLASS = HStoreKey.class;
+ static final Class<? extends Writable> VALUE_CLASS =
+ ImmutableBytesWritable.class;
static class HbaseReader extends MapFile.Reader {
@@ -569,8 +570,6 @@
* @param conf
* @param fs
* @param dirName
- * @param keyClass
- * @param valClass
* @param compression
* @throws IOException
*/
@@ -839,4 +838,4 @@
return super.seek(key);
}
}
-}
+}
\ No newline at end of file