You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2021/05/18 19:44:43 UTC
[incubator-pinot] branch add-two-broker-metrics updated (06cd673 ->
008a925)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch add-two-broker-metrics
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 06cd673 Add SERVER_PROCESSING and NETTY_CONNECTION to broker query phase metrics
new 008a925 Add SERVER_PROCESSING and NETTY_CONNECTION to broker query phase metrics
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (06cd673)
\
N -- N -- N refs/heads/add-two-broker-metrics (008a925)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../requesthandler/SingleConnectionBrokerRequestHandler.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Add SERVER_PROCESSING and NETTY_CONNECTION
to broker query phase metrics
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch add-two-broker-metrics
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 008a92523612e3757bec0bf010446f5ab33ed53c
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Tue May 18 12:30:51 2021 -0700
Add SERVER_PROCESSING and NETTY_CONNECTION to broker query phase metrics
---
.../SingleConnectionBrokerRequestHandler.java | 14 ++++++++++++--
.../org/apache/pinot/common/metrics/BrokerQueryPhase.java | 2 ++
.../pinot/core/common/datatable/DataTableImplV2.java | 1 +
.../pinot/core/common/datatable/DataTableImplV3.java | 1 +
.../org/apache/pinot/core/transport/ServerResponse.java | 7 +++++++
5 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
index 0abf690..518a51a 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java
@@ -87,11 +87,12 @@ public class SingleConnectionBrokerRequestHandler extends BaseBrokerRequestHandl
.submitQuery(requestId, rawTableName, offlineBrokerRequest, offlineRoutingTable, realtimeBrokerRequest,
realtimeRoutingTable, timeoutMs);
Map<ServerRoutingInstance, ServerResponse> response = asyncQueryResponse.getResponse();
- _brokerMetrics
- .addPhaseTiming(rawTableName, BrokerQueryPhase.SCATTER_GATHER, System.nanoTime() - scatterGatherStartTimeNs);
+ long scatterGatherDurationNs = System.nanoTime() - scatterGatherStartTimeNs;
+ _brokerMetrics.addPhaseTiming(rawTableName, BrokerQueryPhase.SCATTER_GATHER, scatterGatherDurationNs);
// TODO Use scatterGatherStats as serverStats
serverStats.setServerStats(asyncQueryResponse.getStats());
+ long serverMaxProcessingTimeMs = 0;
int numServersQueried = response.size();
long totalResponseSize = 0;
Map<ServerRoutingInstance, DataTable> dataTableMap = new HashMap<>(HashUtil.getHashMapCapacity(numServersQueried));
@@ -102,8 +103,17 @@ public class SingleConnectionBrokerRequestHandler extends BaseBrokerRequestHandl
dataTableMap.put(entry.getKey(), dataTable);
totalResponseSize += serverResponse.getResponseSize();
}
+ long serverProcessingTimeMs = serverResponse.getServerProcessingTimeMs();
+ if (serverProcessingTimeMs > -1) {
+ // Get the maximum value of serverProcessingTimeMs across all the server responses.
+ serverMaxProcessingTimeMs = Math.max(serverMaxProcessingTimeMs, serverProcessingTimeMs);
+ }
}
int numServersResponded = dataTableMap.size();
+ long serverMaxProcessingTimeNs = TimeUnit.MILLISECONDS.toNanos(serverMaxProcessingTimeMs);
+ _brokerMetrics.addPhaseTiming(rawTableName, BrokerQueryPhase.SERVER_PROCESSING, serverMaxProcessingTimeNs);
+ _brokerMetrics.addPhaseTiming(rawTableName, BrokerQueryPhase.NETTY_CONNECTION,
+ scatterGatherDurationNs - serverMaxProcessingTimeNs);
long reduceStartTimeNs = System.nanoTime();
long reduceTimeOutMs = timeoutMs - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - scatterGatherStartTimeNs);
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java
index cc8cd58..5a0e449 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java
@@ -30,6 +30,8 @@ public enum BrokerQueryPhase implements AbstractMetrics.QueryPhase {
QUERY_EXECUTION,
QUERY_ROUTING,
SCATTER_GATHER,
+ SERVER_PROCESSING,
+ NETTY_CONNECTION,
DESERIALIZATION,
REDUCE,
REQUEST_CONNECTION_WAIT,
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java
index e1afdd7..9453f01 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java
@@ -55,6 +55,7 @@ public class DataTableImplV2 extends BaseDataTable {
* Construct empty data table. (Server side)
*/
public DataTableImplV2() {
+ super();
}
/**
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
index 56149e4..9cdba9c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
@@ -86,6 +86,7 @@ public class DataTableImplV3 extends BaseDataTable {
* Construct empty data table. (Server side)
*/
public DataTableImplV3() {
+ super();
_errCodeToExceptionMap = new HashMap<>();
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerResponse.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerResponse.java
index 0583884..a685f4c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerResponse.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerResponse.java
@@ -30,6 +30,7 @@ public class ServerResponse {
private final long _startTimeMs;
private volatile long _submitRequestTimeMs;
private volatile long _receiveDataTableTimeMs;
+ private volatile long _serverProcessingTimeMs;
private volatile DataTable _dataTable;
private volatile int _responseSize;
private volatile int _deserializationTimeMs;
@@ -58,6 +59,10 @@ public class ServerResponse {
}
}
+ public long getServerProcessingTimeMs() {
+ return _serverProcessingTimeMs;
+ }
+
public int getResponseSize() {
return _responseSize;
}
@@ -81,5 +86,7 @@ public class ServerResponse {
_dataTable = dataTable;
_responseSize = responseSize;
_deserializationTimeMs = deserializationTimeMs;
+ _serverProcessingTimeMs =
+ Long.parseLong(_dataTable.getMetadata().getOrDefault(DataTable.MetadataKey.TIME_USED_MS.getName(), "-1"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org