You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2015/06/16 13:47:22 UTC
hbase git commit: HBASE-13886 Return empty value when the mob file is
corrupt instead of throwing exceptions. (Jingcheng)
Repository: hbase
Updated Branches:
refs/heads/hbase-11339 2e2742183 -> b88933900
HBASE-13886 Return empty value when the mob file is corrupt instead of throwing exceptions. (Jingcheng)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b8893390
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b8893390
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b8893390
Branch: refs/heads/hbase-11339
Commit: b889339006eff73428ba93f39c7dbe4ab483ff9a
Parents: 2e27421
Author: anoopsjohn <an...@gmail.com>
Authored: Tue Jun 16 17:17:03 2015 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Tue Jun 16 17:17:03 2015 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/mob/MobConstants.java | 1 +
.../org/apache/hadoop/hbase/mob/MobUtils.java | 15 +++++
.../hadoop/hbase/regionserver/HMobStore.java | 39 +++++++++---
.../hbase/regionserver/MobStoreScanner.java | 9 ++-
.../regionserver/ReversedMobStoreScanner.java | 9 ++-
.../hbase/regionserver/TestMobStoreScanner.java | 63 ++++++++++++++++++++
6 files changed, 123 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8893390/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
index dd33cda..4dfb7b6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
@@ -35,6 +35,7 @@ public class MobConstants {
public static final String MOB_SCAN_RAW = "hbase.mob.scan.raw";
public static final String MOB_CACHE_BLOCKS = "hbase.mob.cache.blocks";
public static final String MOB_SCAN_REF_ONLY = "hbase.mob.scan.ref.only";
+ public static final String EMPTY_VALUE_ON_MOBCELL_MISS = "empty.value.on.mobcell.miss";
public static final String MOB_FILE_CACHE_SIZE_KEY = "hbase.mob.file.cache.size";
public static final int DEFAULT_MOB_FILE_CACHE_SIZE = 1000;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8893390/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index c40767c..53cd1a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -855,4 +855,19 @@ public class MobUtils {
}
return false;
}
+
+ /**
+ * Indicates whether return null value when the mob file is missing or corrupt.
+ * The information is set in the attribute "empty.value.on.mobcell.miss" of scan.
+ * @param scan The current scan.
+ * @return True if the readEmptyValueOnMobCellMiss is enabled.
+ */
+ public static boolean isReadEmptyValueOnMobCellMiss(Scan scan) {
+ byte[] readEmptyValueOnMobCellMiss = scan.getAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS);
+ try {
+ return readEmptyValueOnMobCellMiss != null && Bytes.toBoolean(readEmptyValueOnMobCellMiss);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8893390/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index 4a782dd..8f12656 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.master.TableLockManager;
@@ -310,13 +311,14 @@ public class HMobStore extends HStore {
/**
* Reads the cell from the mob file, and the read point does not count.
+ * This is used for DefaultMobStoreCompactor where we can read empty value for the missing cell.
* @param reference The cell found in the HBase, its value is a path to a mob file.
* @param cacheBlocks Whether the scanner should cache blocks.
* @return The cell found in the mob file.
* @throws IOException
*/
public Cell resolve(Cell reference, boolean cacheBlocks) throws IOException {
- return resolve(reference, cacheBlocks, -1);
+ return resolve(reference, cacheBlocks, -1, true);
}
/**
@@ -324,10 +326,13 @@ public class HMobStore extends HStore {
* @param reference The cell found in the HBase, its value is a path to a mob file.
* @param cacheBlocks Whether the scanner should cache blocks.
* @param readPt the read point.
+ * @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
+ * missing or corrupt.
* @return The cell found in the mob file.
* @throws IOException
*/
- public Cell resolve(Cell reference, boolean cacheBlocks, long readPt) throws IOException {
+ public Cell resolve(Cell reference, boolean cacheBlocks, long readPt,
+ boolean readEmptyValueOnMobCellMiss) throws IOException {
Cell result = null;
if (MobUtils.hasValidMobRefCellValue(reference)) {
String fileName = MobUtils.getMobFileName(reference);
@@ -352,7 +357,8 @@ public class HMobStore extends HStore {
keyLock.releaseLockEntry(lockEntry);
}
}
- result = readCell(locations, fileName, reference, cacheBlocks, readPt);
+ result = readCell(locations, fileName, reference, cacheBlocks, readPt,
+ readEmptyValueOnMobCellMiss);
}
}
if (result == null) {
@@ -380,12 +386,15 @@ public class HMobStore extends HStore {
* @param search The cell to be searched.
* @param cacheMobBlocks Whether the scanner should cache blocks.
* @param readPt the read point.
+ * @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
+ * missing or corrupt.
* @return The found cell. Null if there's no such a cell.
* @throws IOException
*/
private Cell readCell(List<Path> locations, String fileName, Cell search, boolean cacheMobBlocks,
- long readPt) throws IOException {
+ long readPt, boolean readEmptyValueOnMobCellMiss) throws IOException {
FileSystem fs = getFileSystem();
+ Throwable throwable = null;
for (Path location : locations) {
MobFile file = null;
Path path = new Path(location, fileName);
@@ -395,27 +404,39 @@ public class HMobStore extends HStore {
cacheMobBlocks);
} catch (IOException e) {
mobCacheConfig.getMobFileCache().evictFile(fileName);
+ throwable = e;
if ((e instanceof FileNotFoundException) ||
(e.getCause() instanceof FileNotFoundException)) {
LOG.warn("Fail to read the cell, the mob file " + path + " doesn't exist", e);
+ } else if (e instanceof CorruptHFileException) {
+ LOG.error("The mob file " + path + " is corrupt", e);
+ break;
} else {
throw e;
}
- } catch (NullPointerException e) {
+ } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
mobCacheConfig.getMobFileCache().evictFile(fileName);
LOG.warn("Fail to read the cell", e);
- } catch (AssertionError e) {
+ throwable = e;
+ } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
mobCacheConfig.getMobFileCache().evictFile(fileName);
LOG.warn("Fail to read the cell", e);
+ throwable = e;
} finally {
if (file != null) {
mobCacheConfig.getMobFileCache().closeFile(file);
}
}
}
- LOG.error("The mob file " + fileName + " could not be found in the locations "
- + locations);
- return null;
+ LOG.error("The mob file " + fileName + " could not be found in the locations " + locations
+ + " or it is corrupt");
+ if (readEmptyValueOnMobCellMiss) {
+ return null;
+ } else if (throwable instanceof IOException) {
+ throw (IOException) throwable;
+ } else {
+ throw new IOException(throwable);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8893390/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java
index 0521cce..46bbfd5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java
@@ -36,12 +36,16 @@ import org.apache.hadoop.hbase.mob.MobUtils;
public class MobStoreScanner extends StoreScanner {
private boolean cacheMobBlocks = false;
+ private boolean rawMobScan = false;
+ private boolean readEmptyValueOnMobCellMiss = false;
private final HMobStore mobStore;
public MobStoreScanner(Store store, ScanInfo scanInfo, Scan scan,
final NavigableSet<byte[]> columns, long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
cacheMobBlocks = MobUtils.isCacheMobBlocks(scan);
+ rawMobScan = MobUtils.isRawMobScan(scan);
+ readEmptyValueOnMobCellMiss = MobUtils.isReadEmptyValueOnMobCellMiss(scan);
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
@@ -56,7 +60,7 @@ public class MobStoreScanner extends StoreScanner {
@Override
public boolean next(List<Cell> outResult, ScannerContext ctx) throws IOException {
boolean result = super.next(outResult, ctx);
- if (!MobUtils.isRawMobScan(scan)) {
+ if (!rawMobScan) {
// retrieve the mob data
if (outResult.isEmpty()) {
return result;
@@ -66,7 +70,8 @@ public class MobStoreScanner extends StoreScanner {
for (int i = 0; i < outResult.size(); i++) {
Cell cell = outResult.get(i);
if (MobUtils.isMobReferenceCell(cell)) {
- Cell mobCell = mobStore.resolve(cell, cacheMobBlocks, readPt);
+ Cell mobCell = mobStore
+ .resolve(cell, cacheMobBlocks, readPt, readEmptyValueOnMobCellMiss);
mobKVCount++;
mobKVSize += mobCell.getValueLength();
outResult.set(i, mobCell);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8893390/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
index aa36e3e..78c1720 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
@@ -36,12 +36,16 @@ import org.apache.hadoop.hbase.mob.MobUtils;
public class ReversedMobStoreScanner extends ReversedStoreScanner {
private boolean cacheMobBlocks = false;
+ private boolean rawMobScan = false;
+ private boolean readEmptyValueOnMobCellMiss = false;
protected final HMobStore mobStore;
ReversedMobStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet<byte[]> columns,
long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
cacheMobBlocks = MobUtils.isCacheMobBlocks(scan);
+ rawMobScan = MobUtils.isRawMobScan(scan);
+ readEmptyValueOnMobCellMiss = MobUtils.isReadEmptyValueOnMobCellMiss(scan);
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
@@ -56,7 +60,7 @@ public class ReversedMobStoreScanner extends ReversedStoreScanner {
@Override
public boolean next(List<Cell> outResult, ScannerContext ctx) throws IOException {
boolean result = super.next(outResult, ctx);
- if (!MobUtils.isRawMobScan(scan)) {
+ if (!rawMobScan) {
// retrieve the mob data
if (outResult.isEmpty()) {
return result;
@@ -66,7 +70,8 @@ public class ReversedMobStoreScanner extends ReversedStoreScanner {
for (int i = 0; i < outResult.size(); i++) {
Cell cell = outResult.get(i);
if (MobUtils.isMobReferenceCell(cell)) {
- Cell mobCell = mobStore.resolve(cell, cacheMobBlocks, readPt);
+ Cell mobCell = mobStore
+ .resolve(cell, cacheMobBlocks, readPt, readEmptyValueOnMobCellMiss);
mobKVCount++;
mobKVSize += mobCell.getValueLength();
outResult.set(i, mobCell);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8893390/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
index 01b6a5f..820087a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -40,6 +41,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
+import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -68,6 +71,8 @@ public class TestMobStoreScanner {
private static HTableDescriptor desc;
private static Random random = new Random();
private static long defaultThreshold = 10;
+ private FileSystem fs;
+ private Configuration conf;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -84,6 +89,8 @@ public class TestMobStoreScanner {
}
public void setUp(long threshold, TableName tn) throws Exception {
+ conf = TEST_UTIL.getConfiguration();
+ fs = FileSystem.get(conf);
desc = new HTableDescriptor(tn);
hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
@@ -196,6 +203,62 @@ public class TestMobStoreScanner {
Assert.assertEquals("value2", Bytes.toString(cell.getValue()));
}
+ @Test
+ public void testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss() throws Exception {
+ TableName tn = TableName.valueOf("testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss");
+ setUp(0, tn);
+ createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
+ Get get = new Get(row1);
+ get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true));
+ Result result = table.get(get);
+ Cell cell = result.getColumnLatestCell(family, qf1);
+ Assert.assertEquals(0, CellUtil.cloneValue(cell).length);
+ }
+
+ @Test
+ public void testReadFromCorruptMobFiles() throws Exception {
+ TableName tn = TableName.valueOf("testReadFromCorruptMobFiles");
+ setUp(0, tn);
+ createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
+ Get get = new Get(row1);
+ IOException ioe = null;
+ try {
+ table.get(get);
+ } catch (IOException e) {
+ ioe = e;
+ }
+ Assert.assertNotNull(ioe);
+ Assert.assertEquals(CorruptHFileException.class.getName(), ioe.getClass().getName());
+ }
+
+ private void createRecordAndCorruptMobFile(TableName tn, byte[] row, byte[] family, byte[] qf,
+ byte[] value) throws IOException {
+ Put put1 = new Put(row);
+ put1.addColumn(family, qf, value);
+ table.put(put1);
+ admin.flush(tn);
+ Path mobFile = getFlushedMobFile(conf, fs, tn, Bytes.toString(family));
+ Assert.assertNotNull(mobFile);
+ // create new corrupt mob file.
+ Path corruptFile = new Path(mobFile.getParent(), "dummy");
+ TestHFile.truncateFile(fs, mobFile, corruptFile);
+ fs.delete(mobFile, true);
+ fs.rename(corruptFile, mobFile);
+ }
+
+ private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family)
+ throws IOException {
+ Path regionDir = MobUtils.getMobRegionPath(conf, table);
+ Path famDir = new Path(regionDir, family);
+ FileStatus[] hfFss = fs.listStatus(famDir);
+ for (FileStatus hfs : hfFss) {
+ if (!hfs.isDirectory()) {
+ return hfs.getPath();
+ }
+ }
+ return null;
+ }
+
private void testGetFromFiles(boolean reversed) throws Exception {
TableName tn = TableName.valueOf("testGetFromFiles" + reversed);
setUp(defaultThreshold, tn);