You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by eo...@apache.org on 2021/11/19 09:48:54 UTC
[bookkeeper] branch master updated: Rejected requests counter metric (#2866)
This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 8720c54 Rejected requests counter metric (#2866)
8720c54 is described below
commit 8720c54904bfe73002abd93b9ae58f7d776cb23d
Author: Jack Vanlightly <va...@gmail.com>
AuthorDate: Fri Nov 19 10:48:48 2021 +0100
Rejected requests counter metric (#2866)
Recording the number of requests that are rejected
due to overload of internal components is important
for diagnosing that a bookie is under stress.
Previously these events were only recorded as request
failures
---
.../bookkeeper/bookie/BookKeeperServerStats.java | 2 ++
.../bookkeeper/proto/BookieRequestProcessor.java | 5 ++++-
.../org/apache/bookkeeper/proto/RequestStats.java | 20 ++++++++++++++++++--
.../apache/bookkeeper/proto/WriteEntryProcessor.java | 1 +
.../bookkeeper/proto/WriteEntryProcessorV3.java | 1 +
5 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
index a5ae48f..d92fea1 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
@@ -46,10 +46,12 @@ public interface BookKeeperServerStats {
// Stats
String ADD_ENTRY_REQUEST = "ADD_ENTRY_REQUEST";
String ADD_ENTRY = "ADD_ENTRY";
+ String ADD_ENTRY_REJECTED = "ADD_ENTRY_REJECTED";
String FORCE_LEDGER_REQUEST = "FORCE_LEDGER_REQUEST";
String FORCE_LEDGER = "FORCE_LEDGER";
String READ_ENTRY_REQUEST = "READ_ENTRY_REQUEST";
String READ_ENTRY = "READ_ENTRY";
+ String READ_ENTRY_REJECTED = "READ_ENTRY_REJECTED";
String READ_ENTRY_SCHEDULING_DELAY = "READ_ENTRY_SCHEDULING_DELAY";
String READ_ENTRY_FENCE_REQUEST = "READ_ENTRY_FENCE_REQUEST";
String READ_ENTRY_FENCE_WAIT = "READ_ENTRY_FENCE_WAIT";
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestProcessor.java
index 67f83e9..dc8f2a3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestProcessor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestProcessor.java
@@ -440,6 +440,7 @@ public class BookieRequestProcessor implements RequestProcessor {
LOG.debug("Failed to process request to add entry at {}:{}. Too many pending requests",
r.getAddRequest().getLedgerId(), r.getAddRequest().getEntryId());
}
+ getRequestStats().getAddEntryRejectedCounter().inc();
BookkeeperProtocol.AddResponse.Builder addResponse = BookkeeperProtocol.AddResponse.newBuilder()
.setLedgerId(r.getAddRequest().getLedgerId())
.setEntryId(r.getAddRequest().getEntryId())
@@ -527,6 +528,7 @@ public class BookieRequestProcessor implements RequestProcessor {
LOG.debug("Failed to process request to read entry at {}:{}. Too many pending requests",
r.getReadRequest().getLedgerId(), r.getReadRequest().getEntryId());
}
+ getRequestStats().getReadEntryRejectedCounter().inc();
BookkeeperProtocol.ReadResponse.Builder readResponse = BookkeeperProtocol.ReadResponse.newBuilder()
.setLedgerId(r.getReadRequest().getLedgerId())
.setEntryId(r.getReadRequest().getEntryId())
@@ -634,6 +636,7 @@ public class BookieRequestProcessor implements RequestProcessor {
LOG.debug("Failed to process request to add entry at {}:{}. Too many pending requests", r.ledgerId,
r.entryId);
}
+ getRequestStats().getAddEntryRejectedCounter().inc();
write.sendResponse(
BookieProtocol.ETOOMANYREQUESTS,
@@ -668,7 +671,7 @@ public class BookieRequestProcessor implements RequestProcessor {
LOG.debug("Failed to process request to read entry at {}:{}. Too many pending requests", r.ledgerId,
r.entryId);
}
-
+ getRequestStats().getReadEntryRejectedCounter().inc();
read.sendResponse(
BookieProtocol.ETOOMANYREQUESTS,
ResponseBuilder.buildErrorResponse(BookieProtocol.ETOOMANYREQUESTS, r),
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/RequestStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/RequestStats.java
index fd91957..b12e153 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/RequestStats.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/RequestStats.java
@@ -23,6 +23,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ADD_ENTRY;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ADD_ENTRY_BLOCKED;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ADD_ENTRY_BLOCKED_WAIT;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ADD_ENTRY_IN_PROGRESS;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ADD_ENTRY_REJECTED;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ADD_ENTRY_REQUEST;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.CHANNEL_WRITE;
@@ -43,6 +44,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_LONG
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_LONG_POLL_READ;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_LONG_POLL_REQUEST;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_LONG_POLL_WAIT;
+import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_REJECTED;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_REQUEST;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_ENTRY_SCHEDULING_DELAY;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.READ_LAC;
@@ -91,6 +93,12 @@ public class RequestStats {
)
private final OpStatsLogger addEntryStats;
@StatsDoc(
+ name = ADD_ENTRY_REJECTED,
+ help = "Counter for rejected adds on a bookie",
+ parent = ADD_ENTRY_REQUEST
+ )
+ private final Counter addEntryRejectedCounter;
+ @StatsDoc(
name = READ_ENTRY_REQUEST,
help = "request stats of ReadEntry on a bookie"
)
@@ -102,6 +110,12 @@ public class RequestStats {
)
final OpStatsLogger readEntryStats;
@StatsDoc(
+ name = READ_ENTRY_REJECTED,
+ help = "Counter for rejected reads on a bookie",
+ parent = READ_ENTRY_REQUEST
+ )
+ private final Counter readEntryRejectedCounter;
+ @StatsDoc(
name = FORCE_LEDGER,
help = "operation stats of ForceLedger on a bookie",
parent = FORCE_LEDGER_REQUEST
@@ -225,9 +239,11 @@ public class RequestStats {
final OpStatsLogger getListOfEntriesOfLedgerStats;
public RequestStats(StatsLogger statsLogger) {
- this.addEntryStats = statsLogger.getOpStatsLogger(ADD_ENTRY);
+ this.addEntryStats = statsLogger.getThreadScopedOpStatsLogger(ADD_ENTRY);
this.addRequestStats = statsLogger.getOpStatsLogger(ADD_ENTRY_REQUEST);
- this.readEntryStats = statsLogger.getOpStatsLogger(READ_ENTRY);
+ this.addEntryRejectedCounter = statsLogger.getCounter(ADD_ENTRY_REJECTED);
+ this.readEntryStats = statsLogger.getThreadScopedOpStatsLogger(READ_ENTRY);
+ this.readEntryRejectedCounter = statsLogger.getCounter(READ_ENTRY_REJECTED);
this.forceLedgerStats = statsLogger.getOpStatsLogger(FORCE_LEDGER);
this.forceLedgerRequestStats = statsLogger.getOpStatsLogger(FORCE_LEDGER_REQUEST);
this.readRequestStats = statsLogger.getOpStatsLogger(READ_ENTRY_REQUEST);
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java
index 7dbd30c..622016b 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java
@@ -80,6 +80,7 @@ class WriteEntryProcessor extends PacketProcessorBase<ParsedAddRequest> implemen
requestProcessor.getBookie().addEntry(addData, false, this, channel, request.getMasterKey());
}
} catch (OperationRejectedException e) {
+ requestProcessor.getRequestStats().getAddEntryRejectedCounter().inc();
// Avoid to log each occurence of this exception as this can happen when the ledger storage is
// unable to keep up with the write rate.
if (LOG.isDebugEnabled()) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java
index ed2e304..597c54c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java
@@ -126,6 +126,7 @@ class WriteEntryProcessorV3 extends PacketProcessorBaseV3 {
}
status = StatusCode.EOK;
} catch (OperationRejectedException e) {
+ requestProcessor.getRequestStats().getAddEntryRejectedCounter().inc();
// Avoid to log each occurence of this exception as this can happen when the ledger storage is
// unable to keep up with the write rate.
if (logger.isDebugEnabled()) {