You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/06/27 01:55:45 UTC
[9/9] hbase git commit: HBASE-16085 Add a metric for failed
compactions
HBASE-16085 Add a metric for failed compactions
Conflicts:
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41f6f7e3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41f6f7e3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41f6f7e3
Branch: refs/heads/0.98
Commit: 41f6f7e30915977f178071fae2ac3b3585347222
Parents: 099b3da
Author: Gary Helmling <ga...@apache.org>
Authored: Wed Jun 22 17:08:28 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jun 24 11:36:22 2016 -0700
----------------------------------------------------------------------
.../hbase/regionserver/MetricsRegionSource.java | 2 +
.../regionserver/MetricsRegionWrapper.java | 7 ++++
.../regionserver/MetricsRegionSourceImpl.java | 3 ++
.../TestMetricsRegionSourceImpl.java | 6 +++
.../regionserver/MetricsRegionSourceImpl.java | 3 ++
.../TestMetricsRegionSourceImpl.java | 5 +++
.../hbase/regionserver/CompactSplitThread.java | 2 +
.../hadoop/hbase/regionserver/HRegion.java | 10 ++++-
.../regionserver/MetricsRegionWrapperImpl.java | 5 +++
.../regionserver/MetricsRegionWrapperStub.java | 5 +++
.../hbase/regionserver/TestCompaction.java | 43 ++++++++++++++++++++
11 files changed, 89 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
index 8abbb1f..cfe95ed 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
@@ -28,9 +28,11 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource> {
String OPS_SAMPLE_NAME = "ops";
String SIZE_VALUE_NAME = "size";
String COMPACTIONS_COMPLETED_COUNT = "compactionsCompletedCount";
+ String COMPACTIONS_FAILED_COUNT = "compactionsFailedCount";
String NUM_BYTES_COMPACTED_COUNT = "numBytesCompactedCount";
String NUM_FILES_COMPACTED_COUNT = "numFilesCompactedCount";
String COMPACTIONS_COMPLETED_DESC = "Number of compactions that have completed.";
+ String COMPACTIONS_FAILED_DESC = "Number of compactions that have failed.";
String NUM_BYTES_COMPACTED_DESC =
"Sum of filesize on all files entering a finished, successful or aborted, compaction";
String NUM_FILES_COMPACTED_DESC =
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 2a3bfb7..5349b5b 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -105,6 +105,13 @@ public interface MetricsRegionWrapper {
long getNumCompactionsCompleted();
/**
+ * Returns the total number of compactions that have been reported as failed on this region.
+ * Note that a given compaction can be reported as both completed and failed if an exception
+ * is thrown in the processing after {@code HRegion.compact()}.
+ */
+ long getNumCompactionsFailed();
+
+ /**
* Get the time spent by coprocessors in this region.
*/
Map<String, DescriptiveStatistics> getCoprocessorExecutionStatistics();
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 358d287..31d7721 100644
--- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -199,6 +199,9 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
mrb.addCounter(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT,
MetricsRegionSource.COMPACTIONS_COMPLETED_DESC,
this.regionWrapper.getNumCompactionsCompleted());
+ mrb.addCounter(regionNamePrefix + MetricsRegionSource.COMPACTIONS_FAILED_COUNT,
+ MetricsRegionSource.COMPACTIONS_FAILED_DESC,
+ this.regionWrapper.getNumCompactionsFailed());
mrb.addCounter(regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT,
MetricsRegionSource.NUM_BYTES_COMPACTED_DESC,
this.regionWrapper.getNumBytesCompacted());
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index aca04c7..c041ffa 100644
--- a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -144,9 +144,15 @@ public class TestMetricsRegionSourceImpl {
public long getNumCompactionsCompleted() {
return 0;
}
+
@Override
public Map<String, DescriptiveStatistics> getCoprocessorExecutionStatistics() {
return null;
}
+
+ @Override
+ public long getNumCompactionsFailed() {
+ return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 8bc87e0..b8a74ca 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -201,6 +201,9 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT,
MetricsRegionSource.NUM_BYTES_COMPACTED_DESC),
this.regionWrapper.getNumBytesCompacted());
+ mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_FAILED_COUNT,
+ MetricsRegionSource.COMPACTIONS_FAILED_DESC),
+ this.regionWrapper.getNumCompactionsFailed());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_FILES_COMPACTED_COUNT,
MetricsRegionSource.NUM_FILES_COMPACTED_DESC),
this.regionWrapper.getNumFilesCompacted());
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index 539e235..5d0ab82 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -149,5 +149,10 @@ public class TestMetricsRegionSourceImpl {
public Map<String, DescriptiveStatistics> getCoprocessorExecutionStatistics() {
return null;
}
+
+ @Override
+ public long getNumCompactionsFailed() {
+ return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 3e73ec1..5246287 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -518,9 +518,11 @@ public class CompactSplitThread implements CompactionRequestor {
if (remoteEx != ex) {
LOG.info("Compaction failed at original callstack: " + formatStackTrace(ex));
}
+ region.reportCompactionRequestFailure();
server.checkFileSystem();
} catch (Exception ex) {
LOG.error("Compaction failed " + this, ex);
+ region.reportCompactionRequestFailure();
server.checkFileSystem();
} finally {
LOG.debug("CompactSplitThread Status: " + CompactSplitThread.this);
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 94e3851..bd2b7e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -291,6 +291,7 @@ public class HRegion implements HeapSize { // , Writable{
// Compaction counters
final AtomicLong compactionsFinished = new AtomicLong(0L);
+ final AtomicLong compactionsFailed = new AtomicLong(0L);
final AtomicLong compactionNumFilesCompacted = new AtomicLong(0L);
final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L);
@@ -5946,7 +5947,7 @@ public class HRegion implements HeapSize { // , Writable{
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
- 42 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+ 43 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
(12 * Bytes.SIZEOF_LONG) +
5 * Bytes.SIZEOF_BOOLEAN);
@@ -5963,7 +5964,8 @@ public class HRegion implements HeapSize { // , Writable{
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
ClassSize.OBJECT + // closeLock
(2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing
- (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL
+ (4 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL,
+ // compactionsFailed
(2 * ClassSize.CONCURRENT_HASHMAP) + // lockedRows, scannerReadPoints
WriteState.HEAP_SIZE + // writestate
ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + // stores
@@ -6570,6 +6572,10 @@ public class HRegion implements HeapSize { // , Writable{
assert newValue >= 0;
}
+ public void reportCompactionRequestFailure() {
+ compactionsFailed.incrementAndGet();
+ }
+
/**
* @return sequenceId.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index 1cdcb0f..d66409a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -140,6 +140,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
}
@Override
+ public long getNumCompactionsFailed() {
+ return this.region.compactionsFailed.get();
+ }
+
+ @Override
public long getMaxStoreFileAge() {
return maxStoreFileAge;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index c41f93a..874f4e2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -109,4 +109,9 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
public Map<String, DescriptiveStatistics> getCoprocessorExecutionStatistics() {
return new HashMap<String, DescriptiveStatistics>();
}
+
+ @Override
+ public long getNumCompactionsFailed() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 9aa0142..c2422e2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -36,6 +36,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -297,6 +298,48 @@ public class TestCompaction {
thread.interruptIfNecessary();
}
+ @Test
+ public void testCompactionFailure() throws Exception {
+ // setup a compact/split thread on a mock server
+ HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+ Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf());
+ CompactSplitThread thread = new CompactSplitThread(mockServer);
+ Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+
+ // setup a region/store with some files
+ Store store = r.getStore(COLUMN_FAMILY);
+ createStoreFile(r);
+ for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) {
+ createStoreFile(r);
+ }
+
+ HRegion mockRegion = Mockito.spy(r);
+ Mockito.when(mockRegion.checkSplit()).thenThrow(new IndexOutOfBoundsException());
+
+ MetricsRegionWrapper metricsWrapper = new MetricsRegionWrapperImpl(r);
+
+ long preCompletedCount = metricsWrapper.getNumCompactionsCompleted();
+ long preFailedCount = metricsWrapper.getNumCompactionsFailed();
+
+ CountDownLatch latch = new CountDownLatch(1);
+ TrackableCompactionRequest request = new TrackableCompactionRequest(latch);
+ thread.requestCompaction(mockRegion, store, "test custom comapction", Store.PRIORITY_USER,
+ request, null);
+ // wait for the latch to complete.
+ latch.await(120, TimeUnit.SECONDS);
+
+ // compaction should have completed and been marked as failed due to error in split request
+ long postCompletedCount = metricsWrapper.getNumCompactionsCompleted();
+ long postFailedCount = metricsWrapper.getNumCompactionsFailed();
+
+ assertTrue("Completed count should have increased (pre=" + preCompletedCount +
+ ", post="+postCompletedCount+")",
+ postCompletedCount > preCompletedCount);
+ assertTrue("Failed count should have increased (pre=" + preFailedCount +
+ ", post=" + postFailedCount + ")",
+ postFailedCount > preFailedCount);
+ }
+
/**
* HBASE-7947: Regression test to ensure adding to the correct list in the
* {@link CompactSplitThread}