You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2012/12/15 20:20:05 UTC
svn commit: r1422319 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/metrics/
test/java/org/apache/hadoop/hbase/regionserver/
Author: liyin
Date: Sat Dec 15 19:20:03 2012
New Revision: 1422319
URL: http://svn.apache.org/viewvc?rev=1422319&view=rev
Log:
[HBASE-7358] Adding Compaction write size metric
Author: manukranthk
Summary:
Adding metric to capture the amount of data(in bytes) that was written during a compaction. This along with the volume of data flushed might give better estimates on the usage patterns. (There was some problem with my
git commit messages so, I was not able to commit to the earlier revision(D653175), hence adding a new revision.)
Test Plan: Adding unit tests.
Reviewers: kannan, liyintang
Reviewed By: kannan
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D657388
Task ID: 1944049
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionConfiguration.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1422319&r1=1422318&r2=1422319&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Sat Dec 15 19:20:03 2012
@@ -265,6 +265,9 @@ public final class HConstants {
/** Default maximum file size */
public static final long DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024;
+
+ /** Default minimum number of files to be compacted */
+ public static final int DEFAULT_MIN_FILES_TO_COMPACT = 3;
/** Default value for files without minFlushTime in metadata */
public static final long NO_MIN_FLUSH_TIME = -1;
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionConfiguration.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionConfiguration.java?rev=1422319&r1=1422318&r2=1422319&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionConfiguration.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionConfiguration.java Sat Dec 15 19:20:03 2012
@@ -71,7 +71,7 @@ public class CompactionConfiguration {
minCompactSize = conf.getLong(strPrefix + "min.size", store.getHRegion().memstoreFlushSize);
shouldExcludeBulk = conf.getBoolean(strPrefix + "exclude.bulk", false);
minFilesToCompact = Math.max(2, conf.getInt(strPrefix + "min",
- /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
+ /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", HConstants.DEFAULT_MIN_FILES_TO_COMPACT)));
maxFilesToCompact = conf.getInt(strPrefix + "max", 10);
compactionRatio = conf.getFloat(strPrefix + "ratio", 1.2F);
offPeekCompactionRatio = conf.getFloat(strPrefix + "ratio.offpeak", 5.0F);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1422319&r1=1422318&r2=1422319&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Sat Dec 15 19:20:03 2012
@@ -1231,6 +1231,9 @@ public class Store extends SchemaConfigu
if (Store.closeCheckInterval > 0) {
bytesWritten += kv.getLength();
if (bytesWritten > Store.closeCheckInterval) {
+ getSchemaMetrics().updatePersistentStoreMetric(
+ SchemaMetrics.StoreMetricType.COMPACTION_WRITE_SIZE,
+ bytesWritten);
bytesWritten = 0;
if (!this.region.areWritesEnabled()) {
writer.close();
@@ -1246,6 +1249,8 @@ public class Store extends SchemaConfigu
}
kvs.clear();
} while (hasMore);
+ getSchemaMetrics().updatePersistentStoreMetric(
+ SchemaMetrics.StoreMetricType.COMPACTION_WRITE_SIZE, bytesWritten);
} finally {
if (scanner != null) {
scanner.close();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java?rev=1422319&r1=1422318&r2=1422319&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java Sat Dec 15 19:20:03 2012
@@ -172,7 +172,8 @@ public class SchemaMetrics {
STATIC_BLOOM_SIZE_KB("staticBloomSizeKB"),
MEMSTORE_SIZE_MB("memstoreSizeMB"),
STATIC_INDEX_SIZE_KB("staticIndexSizeKB"),
- FLUSH_SIZE("flushSize", PERSISTENT_METRIC_FLAG);
+ FLUSH_SIZE("flushSize", PERSISTENT_METRIC_FLAG),
+ COMPACTION_WRITE_SIZE("compactionWriteSize", PERSISTENT_METRIC_FLAG);
private final String metricStr;
private final int flags;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1422319&r1=1422318&r2=1422319&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Sat Dec 15 19:20:03 2012
@@ -30,8 +30,10 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTestConst;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
@@ -290,4 +292,63 @@ public class TestRegionServerMetrics {
testUtil.dropDefaultTable();
}
}
-}
+
+ @Test
+ public void testCompactionWriteMetric() throws Exception {
+ long ts = 2342;
+ String cfName = "cf1";
+ byte[] CF = Bytes.toBytes(cfName);
+ String tableName = "CompactionWriteSize";
+ byte[] TABLE = Bytes.toBytes(tableName);
+ HTable hTable = testUtil.createTable(TABLE, CF);
+
+ final String storeMetricName = ALL_METRICS.getStoreMetricName(
+ StoreMetricType.COMPACTION_WRITE_SIZE);
+ String storeMetricFullName =
+ SchemaMetrics.generateSchemaMetricsPrefix(tableName, cfName)
+ + storeMetricName;
+ long startValue = HRegion.getNumericPersistentMetric(storeMetricFullName);
+
+ int compactionThreshold = conf.getInt("hbase.hstore.compaction.min",
+ HConstants.DEFAULT_MIN_FILES_TO_COMPACT);
+ for (int i=0; i<=compactionThreshold; i++) {
+ String rowName = "row" + i;
+ byte[] ROW = Bytes.toBytes(rowName);
+ Put p = new Put(ROW);
+ p.add(CF, CF, ts, ROW);
+ hTable.put(p);
+ for (HRegion hr : testUtil.getMiniHBaseCluster().getRegions(TABLE)) {
+ hr.flushcache();
+ }
+ }
+
+ for (HRegion hr : testUtil.getMiniHBaseCluster().getRegions(TABLE)) {
+ hr.flushcache();
+ hr.compactStores();
+ }
+
+ long compactionWriteSizeAfterCompaction =
+ HRegion.getNumericPersistentMetric(storeMetricFullName);
+ Assert.assertTrue(startValue < compactionWriteSizeAfterCompaction);
+
+ for (int i=0; i<=compactionThreshold; i++) {
+ String rowName = "row" + i;
+ byte[] ROW = Bytes.toBytes(rowName);
+ Delete del = new Delete(ROW);
+ hTable.delete(del);
+ for (HRegion hr : testUtil.getMiniHBaseCluster().getRegions(TABLE)) {
+ hr.flushcache();
+ }
+ }
+
+ for (HRegion hr : testUtil.getMiniHBaseCluster().getRegions(TABLE)) {
+ hr.flushcache();
+ hr.compactStores();
+ }
+
+ long compactionWriteSizeAfterCompactionAfterDelete =
+ HRegion.getNumericPersistentMetric(storeMetricFullName);
+ Assert.assertTrue(compactionWriteSizeAfterCompactionAfterDelete
+ == compactionWriteSizeAfterCompaction);
+ }
+}
\ No newline at end of file