You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2013/06/15 06:23:25 UTC
svn commit: r1493307 - in /hbase/trunk:
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/
hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionse...
Author: stack
Date: Sat Jun 15 04:23:24 2013
New Revision: 1493307
URL: http://svn.apache.org/r1493307
Log:
HBASE-8689 Cover all mutations rather than only Put while reporting for mutations not writing to WAL
Modified:
hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
hbase/trunk/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
hbase/trunk/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
Modified: hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java (original)
+++ hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java Sat Jun 15 04:23:24 2013
@@ -145,10 +145,10 @@ public interface MetricsRegionServerSour
static final String STATIC_BLOOM_SIZE = "staticBloomSize";
static final String STATIC_BLOOM_SIZE_DESC =
"Uncompressed size of the static bloom filters.";
- static final String NUMBER_OF_PUTS_WITHOUT_WAL = "putsWithoutWALCount";
- static final String NUMBER_OF_PUTS_WITHOUT_WAL_DESC =
+ static final String NUMBER_OF_MUTATIONS_WITHOUT_WAL = "mutationsWithoutWALCount";
+ static final String NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC =
"Number of mutations that have been sent by clients with the write ahead logging turned off.";
- static final String DATA_SIZE_WITHOUT_WAL = "putsWithoutWALSize";
+ static final String DATA_SIZE_WITHOUT_WAL = "mutationsWithoutWALSize";
static final String DATA_SIZE_WITHOUT_WAL_DESC =
"Size of data that has been sent by clients with the write ahead logging turned off.";
static final String PERCENT_FILES_LOCAL = "percentFilesLocal";
Modified: hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java (original)
+++ hbase/trunk/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java Sat Jun 15 04:23:24 2013
@@ -130,7 +130,7 @@ public interface MetricsRegionServerWrap
/**
* Number of mutations received with WAL explicitly turned off.
*/
- long getNumPutsWithoutWAL();
+ long getNumMutationsWithoutWAL();
/**
* Ammount of data in the memstore but not in the WAL because mutations explicitly had their
Modified: hbase/trunk/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (original)
+++ hbase/trunk/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java Sat Jun 15 04:23:24 2013
@@ -164,9 +164,9 @@ public class MetricsRegionServerSourceIm
.addGauge(STOREFILE_INDEX_SIZE, STOREFILE_INDEX_SIZE_DESC, rsWrap.getStoreFileIndexSize())
.addGauge(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC, rsWrap.getTotalStaticIndexSize())
.addGauge(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC, rsWrap.getTotalStaticBloomSize())
- .addGauge(NUMBER_OF_PUTS_WITHOUT_WAL,
- NUMBER_OF_PUTS_WITHOUT_WAL_DESC,
- rsWrap.getNumPutsWithoutWAL())
+ .addGauge(NUMBER_OF_MUTATIONS_WITHOUT_WAL,
+ NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC,
+ rsWrap.getNumMutationsWithoutWAL())
.addGauge(DATA_SIZE_WITHOUT_WAL,
DATA_SIZE_WITHOUT_WAL_DESC,
rsWrap.getDataInMemoryWithoutWAL())
Modified: hbase/trunk/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java (original)
+++ hbase/trunk/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java Sat Jun 15 04:23:24 2013
@@ -173,9 +173,10 @@ public class MetricsRegionServerSourceIm
.addGauge(Interns.info(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC),
rsWrap.getTotalStaticIndexSize())
.addGauge(Interns.info(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC),
- rsWrap.getTotalStaticBloomSize())
- .addGauge(Interns.info(NUMBER_OF_PUTS_WITHOUT_WAL, NUMBER_OF_PUTS_WITHOUT_WAL_DESC),
- rsWrap.getNumPutsWithoutWAL())
+ rsWrap.getTotalStaticBloomSize())
+ .addGauge(
+ Interns.info(NUMBER_OF_MUTATIONS_WITHOUT_WAL, NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC),
+ rsWrap.getNumMutationsWithoutWAL())
.addGauge(Interns.info(DATA_SIZE_WITHOUT_WAL, DATA_SIZE_WITHOUT_WAL_DESC),
rsWrap.getDataInMemoryWithoutWAL())
.addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat Jun 15 04:23:24 2013
@@ -232,7 +232,7 @@ public class HRegion implements HeapSize
public final AtomicLong memstoreSize = new AtomicLong(0);
// Debug possible data loss due to WAL off
- final Counter numPutsWithoutWAL = new Counter();
+ final Counter numMutationsWithoutWAL = new Counter();
final Counter dataInMemoryWithoutWAL = new Counter();
// Debug why CAS operations are taking a while.
@@ -502,7 +502,6 @@ public class HRegion implements HeapSize
// When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled.
this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval",
1 * 1000) <= 0;
-
if (rsServices != null) {
this.rsAccounting = this.rsServices.getRegionServerAccounting();
// don't initialize coprocessors if not running within a regionserver
@@ -1320,8 +1319,8 @@ public class HRegion implements HeapSize
status.setStatus("Running coprocessor pre-flush hooks");
coprocessorHost.preFlush();
}
- if (numPutsWithoutWAL.get() > 0) {
- numPutsWithoutWAL.set(0);
+ if (numMutationsWithoutWAL.get() > 0) {
+ numMutationsWithoutWAL.set(0);
dataInMemoryWithoutWAL.set(0);
}
synchronized (writestate) {
@@ -2177,9 +2176,7 @@ public class HRegion implements HeapSize
durability = tmpDur;
}
if (tmpDur == Durability.SKIP_WAL) {
- if (m instanceof Put) {
- recordPutWithoutWal(m.getFamilyMap());
- }
+ recordMutationWithoutWal(m.getFamilyMap());
continue;
}
@@ -4822,10 +4819,11 @@ public class HRegion implements HeapSize
// Using default cluster id, as this can only happen in the orginating
// cluster. A slave cluster receives the final value (not the delta)
// as a Put.
- txid = this.log.appendNoSync(this.getRegionInfo(),
- this.htableDescriptor.getName(), walEdits,
- HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
- this.htableDescriptor);
+ txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(),
+ walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
+ this.htableDescriptor);
+ } else {
+ recordMutationWithoutWal(append.getFamilyMap());
}
//Actually write to Memstore now
@@ -4970,8 +4968,9 @@ public class HRegion implements HeapSize
txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getName(),
walEdits, HConstants.DEFAULT_CLUSTER_ID, EnvironmentEdgeManager.currentTimeMillis(),
this.htableDescriptor);
+ } else {
+ recordMutationWithoutWal(increment.getFamilyMap());
}
-
//Actually write to Memstore now
for (Map.Entry<Store, List<KeyValue>> entry : tempMemstore.entrySet()) {
Store store = entry.getKey();
@@ -5398,22 +5397,22 @@ public class HRegion implements HeapSize
* Update counters for numer of puts without wal and the size of possible data loss.
* These information are exposed by the region server metrics.
*/
- private void recordPutWithoutWal(final Map<byte [], List<? extends Cell>> familyMap) {
- numPutsWithoutWAL.increment();
- if (numPutsWithoutWAL.get() <= 1) {
+ private void recordMutationWithoutWal(final Map<byte [], List<? extends Cell>> familyMap) {
+ numMutationsWithoutWAL.increment();
+ if (numMutationsWithoutWAL.get() <= 1) {
LOG.info("writing data to region " + this +
" with WAL disabled. Data may be lost in the event of a crash.");
}
- long putSize = 0;
+ long mutationSize = 0;
for (List<? extends Cell> cells: familyMap.values()) {
for (Cell cell : cells) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
- putSize += kv.getKeyLength() + kv.getValueLength();
+ mutationSize += kv.getKeyLength() + kv.getValueLength();
}
}
- dataInMemoryWithoutWAL.add(putSize);
+ dataInMemoryWithoutWAL.add(mutationSize);
}
private void lock(final Lock lock)
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java Sat Jun 15 04:23:24 2013
@@ -62,7 +62,7 @@ class MetricsRegionServerWrapperImpl
private volatile long storefileIndexSize = 0;
private volatile long totalStaticIndexSize = 0;
private volatile long totalStaticBloomSize = 0;
- private volatile long numPutsWithoutWAL = 0;
+ private volatile long numMutationsWithoutWAL = 0;
private volatile long dataInMemoryWithoutWAL = 0;
private volatile int percentFileLocal = 0;
@@ -292,8 +292,8 @@ class MetricsRegionServerWrapperImpl
}
@Override
- public long getNumPutsWithoutWAL() {
- return numPutsWithoutWAL;
+ public long getNumMutationsWithoutWAL() {
+ return numMutationsWithoutWAL;
}
@Override
@@ -344,13 +344,13 @@ class MetricsRegionServerWrapperImpl
long tempStorefileIndexSize = 0;
long tempTotalStaticIndexSize = 0;
long tempTotalStaticBloomSize = 0;
- long tempNumPutsWithoutWAL = 0;
+ long tempNumMutationsWithoutWAL = 0;
long tempDataInMemoryWithoutWAL = 0;
int tempPercentFileLocal = 0;
for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
- tempNumPutsWithoutWAL += r.numPutsWithoutWAL.get();
+ tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get();
tempDataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
tempReadRequestsCount += r.readRequestsCount.get();
tempWriteRequestsCount += r.writeRequestsCount.get();
@@ -404,7 +404,7 @@ class MetricsRegionServerWrapperImpl
storefileIndexSize = tempStorefileIndexSize;
totalStaticIndexSize = tempTotalStaticIndexSize;
totalStaticBloomSize = tempTotalStaticBloomSize;
- numPutsWithoutWAL = tempNumPutsWithoutWAL;
+ numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
percentFileLocal = tempPercentFileLocal;
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java Sat Jun 15 04:23:24 2013
@@ -116,7 +116,7 @@ public class MetricsRegionServerWrapperS
}
@Override
- public long getNumPutsWithoutWAL() {
+ public long getNumMutationsWithoutWAL() {
return 409;
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java Sat Jun 15 04:23:24 2013
@@ -70,8 +70,8 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("storeFileIndexSize", 406, serverSource);
HELPER.assertGauge("staticIndexSize", 407, serverSource);
HELPER.assertGauge("staticBloomSize", 408, serverSource);
- HELPER.assertGauge("putsWithoutWALCount", 409, serverSource);
- HELPER.assertGauge("putsWithoutWALSize", 410, serverSource);
+ HELPER.assertGauge("mutationsWithoutWALCount", 409, serverSource);
+ HELPER.assertGauge("mutationsWithoutWALSize", 410, serverSource);
HELPER.assertGauge("percentFilesLocal", 99, serverSource);
HELPER.assertGauge("compactionQueueLength", 411, serverSource);
HELPER.assertGauge("flushQueueLength", 412, serverSource);
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1493307&r1=1493306&r2=1493307&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Sat Jun 15 04:23:24 2013
@@ -156,8 +156,8 @@ public class TestRegionServerMetrics {
}
@Test
- public void testPutsWithoutWal() throws Exception {
- byte[] tableName = Bytes.toBytes("testPutsWithoutWal");
+ public void testMutationsWithoutWal() throws Exception {
+ byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
@@ -177,9 +177,9 @@ public class TestRegionServerMetrics {
t.flushCommits();
metricsRegionServer.getRegionServerWrapper().forceRecompute();
- metricsHelper.assertGauge("putsWithoutWALCount", 1, serverSource);
+ metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
long minLength = row.length + cf.length + qualifier.length + val.length;
- metricsHelper.assertGaugeGt("putsWithoutWALSize", minLength, serverSource);
+ metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
t.close();
}