You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2014/12/25 23:01:26 UTC
hbase git commit: HBASE-12698 Add mob cell count to the metadata of
each mob file (Jingcheng Du)
Repository: hbase
Updated Branches:
refs/heads/hbase-11339 9f1f8c3bc -> 917adbf0e
HBASE-12698 Add mob cell count to the metadata of each mob file (Jingcheng Du)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/917adbf0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/917adbf0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/917adbf0
Branch: refs/heads/hbase-11339
Commit: 917adbf0e53903e986ed2270a6b270495428b0fb
Parents: 9f1f8c3
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Thu Dec 25 13:59:53 2014 -0800
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Thu Dec 25 13:59:53 2014 -0800
----------------------------------------------------------------------
.../hadoop/hbase/mob/DefaultMobCompactor.java | 3 ++-
.../hbase/mob/DefaultMobStoreFlusher.java | 2 +-
.../hbase/mob/mapreduce/MemStoreWrapper.java | 10 +++-----
.../hadoop/hbase/regionserver/StoreFile.java | 19 +++++++++++++++
.../hbase/regionserver/TestMobCompaction.java | 25 ++++++++++++++++++++
5 files changed, 50 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/917adbf0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
index d0c8175..47dd0b18 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
@@ -219,7 +219,8 @@ public class DefaultMobCompactor extends DefaultCompactor {
} while (hasMore);
} finally {
if (mobFileWriter != null) {
- appendMetadataAndCloseWriter(mobFileWriter, fd, major);
+ mobFileWriter.appendMetadata(fd.maxSeqId, major, mobCells);
+ mobFileWriter.close();
}
}
if(mobFileWriter!=null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/917adbf0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index 194e898..00b3421 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
@@ -196,7 +196,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
} while (hasMore);
} finally {
status.setStatus("Flushing mob file " + store + ": appending metadata");
- mobFileWriter.appendMetadata(cacheFlushId, false);
+ mobFileWriter.appendMetadata(cacheFlushId, false, mobCount);
status.setStatus("Flushing mob file " + store + ": closing flushed file");
mobFileWriter.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/917adbf0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
index b0d4c9d..d286b72 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
@@ -122,7 +122,7 @@ public class MemStoreWrapper {
*/
private void internalFlushCache(final MemStoreSnapshot snapshot)
throws IOException {
- if (snapshot.getSize() == 0) {
+ if (snapshot.getCellsCount() == 0) {
return;
}
// generate the files into a temp directory.
@@ -135,18 +135,16 @@ public class MemStoreWrapper {
LOG.info("Create files under a temp directory " + mobFileWriter.getPath().toString());
byte[] referenceValue = Bytes.toBytes(relativePath);
- int keyValueCount = 0;
KeyValueScanner scanner = snapshot.getScanner();
Cell cell = null;
while (null != (cell = scanner.next())) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
mobFileWriter.append(kv);
- keyValueCount++;
}
scanner.close();
// Write out the log sequence number that corresponds to this output
// hfile. The hfile is current up to and including logCacheFlushId.
- mobFileWriter.appendMetadata(Long.MAX_VALUE, false);
+ mobFileWriter.appendMetadata(Long.MAX_VALUE, false, snapshot.getCellsCount());
mobFileWriter.close();
MobUtils.commitFile(conf, fs, mobFileWriter.getPath(), mobFamilyDir, cacheConfig);
@@ -164,9 +162,7 @@ public class MemStoreWrapper {
table.put(put);
context.getCounter(SweepCounter.RECORDS_UPDATED).increment(1);
}
- if (keyValueCount > 0) {
- table.flushCommits();
- }
+ table.flushCommits();
scanner.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/917adbf0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 13ded58..56ae51a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -112,6 +112,9 @@ public class StoreFile {
/** Key for timestamp of earliest-put in metadata*/
public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
+ /** Key for the number of mob cells in metadata*/
+ public static final byte[] MOB_CELLS_COUNT = Bytes.toBytes("MOB_CELLS_COUNT");
+
private final StoreFileInfo fileInfo;
private final FileSystem fs;
@@ -780,6 +783,22 @@ public class StoreFile {
}
/**
+ * Writes meta data.
+ * Call before {@link #close()} since its written as meta data to this file.
+ * @param maxSequenceId Maximum sequence id.
+ * @param majorCompaction True if this file is product of a major compaction
+ * @param mobCellsCount The number of mob cells.
+ * @throws IOException problem writing to FS
+ */
+ public void appendMetadata(final long maxSequenceId, final boolean majorCompaction,
+ final long mobCellsCount) throws IOException {
+ writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId));
+ writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction));
+ writer.appendFileInfo(MOB_CELLS_COUNT, Bytes.toBytes(mobCellsCount));
+ appendTrackedTimestampsToMetadata();
+ }
+
+ /**
* Add TimestampRange and earliest put timestamp to Metadata
*/
public void appendTrackedTimestampsToMetadata() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/917adbf0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
index f44e529..3234628 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -169,6 +170,8 @@ public class TestMobCompaction {
assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles());
assertEquals("Before compaction: rows", compactionThreshold, countRows());
assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
+ assertEquals("Before compaction: number of mob cells", compactionThreshold,
+ countMobCellsInMetadata());
// Change the threshold larger than the data size
region.getTableDesc().getFamily(COLUMN_FAMILY).setMobThreshold(500);
region.initialize();
@@ -217,6 +220,8 @@ public class TestMobCompaction {
assertEquals("After compaction: rows", compactionThreshold, countRows());
assertEquals("After compaction: mob rows", compactionThreshold, countMobRows());
assertEquals("After compaction: referenced mob file count", 1, countReferencedMobFiles());
+ assertEquals("After compaction: number of mob cells", compactionThreshold,
+ countMobCellsInMetadata());
}
/**
@@ -290,6 +295,26 @@ public class TestMobCompaction {
return 0;
}
+ private long countMobCellsInMetadata() throws IOException {
+ long mobCellsCount = 0;
+ Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
+ hcd.getNameAsString());
+ Configuration copyOfConf = new Configuration(conf);
+ copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
+ CacheConfig cacheConfig = new CacheConfig(copyOfConf);
+ if (fs.exists(mobDirPath)) {
+ FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
+ for (FileStatus file : files) {
+ StoreFile sf = new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE);
+ Map<byte[], byte[]> fileInfo = sf.createReader().loadFileInfo();
+ byte[] count = fileInfo.get(StoreFile.MOB_CELLS_COUNT);
+ assertTrue(count != null);
+ mobCellsCount += Bytes.toLong(count);
+ }
+ }
+ return mobCellsCount;
+ }
+
private Put createPut(int rowIdx, byte[] dummyData) throws IOException {
Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)));
p.setDurability(Durability.SKIP_WAL);