You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2020/08/21 21:08:13 UTC
[incubator-pinot] branch master updated: Expose ResultSetStats in
the Pinot client's ResultSetGroup (#5892)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 331b874 Expose ResultSetStats in the Pinot client's ResultSetGroup (#5892)
331b874 is described below
commit 331b874cd68b9a68b91d20b694bce63bcb5a5e2b
Author: Buchi Reddy Busi Reddy <ma...@gmail.com>
AuthorDate: Fri Aug 21 14:08:02 2020 -0700
Expose ResultSetStats in the Pinot client's ResultSetGroup (#5892)
This would let the client application to observe and print query execution stats selectively for queries and avoid reproducing slow queries.
Changes for the feature in #5871
---
.../org/apache/pinot/client/BrokerResponse.java | 6 +
.../java/org/apache/pinot/client/Connection.java | 2 +-
.../org/apache/pinot/client/ExecutionStats.java | 142 +++++++++++++++++++++
.../org/apache/pinot/client/ResultSetGroup.java | 8 ++
.../apache/pinot/client/PreparedStatementTest.java | 4 +-
.../apache/pinot/client/ResultSetGroupTest.java | 12 +-
6 files changed, 167 insertions(+), 7 deletions(-)
diff --git a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerResponse.java b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerResponse.java
index 3eeb1cd..edac1e9 100644
--- a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerResponse.java
+++ b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/BrokerResponse.java
@@ -29,6 +29,7 @@ class BrokerResponse {
private JsonNode _selectionResults;
private JsonNode _resultTable;
private JsonNode _exceptions;
+ private ExecutionStats _executionStats;
private BrokerResponse() {
}
@@ -38,6 +39,7 @@ class BrokerResponse {
_exceptions = brokerResponse.get("exceptions");
_selectionResults = brokerResponse.get("selectionResults");
_resultTable = brokerResponse.get("resultTable");
+ _executionStats = ExecutionStats.fromJson(brokerResponse);
}
boolean hasExceptions() {
@@ -68,6 +70,10 @@ class BrokerResponse {
}
}
+ ExecutionStats getExecutionStats() {
+ return _executionStats;
+ }
+
static BrokerResponse fromJson(JsonNode json) {
return new BrokerResponse(json);
}
diff --git a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/Connection.java b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/Connection.java
index d926b6c..d408ad5 100644
--- a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/Connection.java
+++ b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/Connection.java
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
public class Connection {
private static final Logger LOGGER = LoggerFactory.getLogger(Connection.class);
private final PinotClientTransport _transport;
- private BrokerSelector _brokerSelector;
+ private final BrokerSelector _brokerSelector;
private List<String> _brokerList;
Connection(List<String> brokerList, PinotClientTransport transport) {
diff --git a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ExecutionStats.java b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ExecutionStats.java
new file mode 100644
index 0000000..0ad420c
--- /dev/null
+++ b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ExecutionStats.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.client;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple POJO to hold query execution statistics for a request. These stats come in every
+ * query that's executed and can be used for debugging Pinot slow queries.
+ *
+ * <p>Please note that objects of this class will hold a reference to the given JsonNode object
+ * and that will only be released when the object is GC'ed.</p>
+ */
+class ExecutionStats {
+
+ private static final String NUM_SERVERS_QUERIED = "numServersQueried";
+ private static final String NUM_SERVERS_RESPONDED = "numServersResponded";
+ private static final String NUM_DOCS_SCANNED = "numDocsScanned";
+ private static final String NUM_ENTRIES_SCANNED_IN_FILTER = "numEntriesScannedInFilter";
+ private static final String NUM_ENTRIES_SCANNED_POST_FILTER = "numEntriesScannedPostFilter";
+ private static final String NUM_SEGMENTS_QUERIED = "numSegmentsQueried";
+ private static final String NUM_SEGMENTS_PROCESSED = "numSegmentsProcessed";
+ private static final String NUM_SEGMENTS_MATCHED = "numSegmentsMatched";
+ private static final String NUM_CONSUMING_SEGMENTS_QUERIED = "numConsumingSegmentsQueried";
+ private static final String MIN_CONSUMING_FRESHNESS_TIME_MS = "minConsumingFreshnessTimeMs";
+ private static final String TOTAL_DOCS = "totalDocs";
+ private static final String NUM_GROUPS_LIMIT_REACHED = "numGroupsLimitReached";
+ private static final String TIME_USED_MS = "timeUsedMs";
+
+ private final JsonNode brokerResponse;
+
+ ExecutionStats(JsonNode brokerResponse) {
+ this.brokerResponse = brokerResponse;
+ }
+
+ static ExecutionStats fromJson(JsonNode json) {
+ return new ExecutionStats(json);
+ }
+
+ public int getNumServersQueried() {
+ // Lazily load the field from the JsonNode to avoid reading the stats when not needed.
+ return brokerResponse.has(NUM_SERVERS_QUERIED) ?
+ brokerResponse.get(NUM_SERVERS_QUERIED).asInt() : -1;
+ }
+
+ public int getNumServersResponded() {
+ return brokerResponse.has(NUM_SERVERS_RESPONDED) ?
+ brokerResponse.get(NUM_SERVERS_RESPONDED).asInt() : -1;
+ }
+
+ public long getNumDocsScanned() {
+ return brokerResponse.has(NUM_DOCS_SCANNED) ?
+ brokerResponse.get(NUM_DOCS_SCANNED).asLong() : -1L;
+ }
+
+ public long getNumEntriesScannedInFilter() {
+ return brokerResponse.has(NUM_ENTRIES_SCANNED_IN_FILTER) ?
+ brokerResponse.get(NUM_ENTRIES_SCANNED_IN_FILTER).asLong() : -1L;
+ }
+
+ public long getNumEntriesScannedPostFilter() {
+ return brokerResponse.has(NUM_ENTRIES_SCANNED_POST_FILTER) ?
+ brokerResponse.get(NUM_ENTRIES_SCANNED_POST_FILTER).asLong() : -1L;
+ }
+
+ public long getNumSegmentsQueried() {
+ return brokerResponse.has(NUM_SEGMENTS_QUERIED) ?
+ brokerResponse.get(NUM_SEGMENTS_QUERIED).asLong() : -1L;
+ }
+
+ public long getNumSegmentsProcessed() {
+ return brokerResponse.has(NUM_SEGMENTS_PROCESSED) ?
+ brokerResponse.get(NUM_SEGMENTS_PROCESSED).asLong() : -1L;
+ }
+
+ public long getNumSegmentsMatched() {
+ return brokerResponse.has(NUM_SEGMENTS_MATCHED) ?
+ brokerResponse.get(NUM_SEGMENTS_MATCHED).asLong() : -1L;
+ }
+
+ public long getNumConsumingSegmentsQueried() {
+ return brokerResponse.has(NUM_CONSUMING_SEGMENTS_QUERIED) ?
+ brokerResponse.get(NUM_CONSUMING_SEGMENTS_QUERIED).asLong() : -1L;
+ }
+
+ public long getMinConsumingFreshnessTimeMs() {
+ return brokerResponse.has(MIN_CONSUMING_FRESHNESS_TIME_MS) ?
+ brokerResponse.get(MIN_CONSUMING_FRESHNESS_TIME_MS).asLong() : -1L;
+ }
+
+ public long getTotalDocs() {
+ return brokerResponse.has(TOTAL_DOCS) ?
+ brokerResponse.get(TOTAL_DOCS).asLong() : -1L;
+ }
+
+ public boolean isNumGroupsLimitReached() {
+ return brokerResponse.has(NUM_GROUPS_LIMIT_REACHED)
+ && brokerResponse.get(NUM_GROUPS_LIMIT_REACHED).asBoolean();
+ }
+
+ public long getTimeUsedMs() {
+ return brokerResponse.has(TIME_USED_MS) ?
+ brokerResponse.get(TIME_USED_MS).asLong() : -1L;
+ }
+
+ @Override
+ public String toString() {
+ Map<String, Object> map = new HashMap<>();
+ map.put(NUM_SERVERS_QUERIED, getNumServersQueried());
+ map.put(NUM_SERVERS_RESPONDED, getNumServersResponded());
+ map.put(NUM_DOCS_SCANNED, getNumDocsScanned());
+ map.put(NUM_ENTRIES_SCANNED_IN_FILTER, getNumEntriesScannedInFilter());
+ map.put(NUM_ENTRIES_SCANNED_POST_FILTER, getNumEntriesScannedPostFilter());
+ map.put(NUM_SEGMENTS_QUERIED, getNumSegmentsQueried());
+ map.put(NUM_SEGMENTS_PROCESSED, getNumSegmentsProcessed());
+ map.put(NUM_SEGMENTS_MATCHED, getNumSegmentsMatched());
+ map.put(NUM_CONSUMING_SEGMENTS_QUERIED, getNumConsumingSegmentsQueried());
+ map.put(MIN_CONSUMING_FRESHNESS_TIME_MS, getMinConsumingFreshnessTimeMs() + "ms");
+ map.put(TOTAL_DOCS, getTotalDocs());
+ map.put(NUM_GROUPS_LIMIT_REACHED, isNumGroupsLimitReached());
+ map.put(TIME_USED_MS, getTimeUsedMs() + "ms");
+ return map.toString();
+ }
+}
diff --git a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSetGroup.java b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSetGroup.java
index e240514..67c4003 100644
--- a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSetGroup.java
+++ b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSetGroup.java
@@ -28,6 +28,7 @@ import java.util.List;
*/
public class ResultSetGroup {
private final List<ResultSet> _resultSets;
+ private final ExecutionStats _executionStats;
ResultSetGroup(BrokerResponse brokerResponse) {
_resultSets = new ArrayList<>();
@@ -52,6 +53,8 @@ public class ResultSetGroup {
}
}
}
+
+ _executionStats = brokerResponse.getExecutionStats();
}
/**
@@ -74,6 +77,10 @@ public class ResultSetGroup {
return _resultSets.get(index);
}
+ public ExecutionStats getExecutionStats() {
+ return _executionStats;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -81,6 +88,7 @@ public class ResultSetGroup {
sb.append(resultSet);
sb.append("\n");
}
+ sb.append(_executionStats.toString());
return sb.toString();
}
}
diff --git a/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/PreparedStatementTest.java b/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/PreparedStatementTest.java
index 4215e77..593f0ed 100644
--- a/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/PreparedStatementTest.java
+++ b/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/PreparedStatementTest.java
@@ -30,7 +30,7 @@ import org.testng.annotations.Test;
*
*/
public class PreparedStatementTest {
- private DummyPinotClientTransport _dummyPinotClientTransport = new DummyPinotClientTransport();
+ private final DummyPinotClientTransport _dummyPinotClientTransport = new DummyPinotClientTransport();
private PinotClientTransportFactory _previousTransportFactory = null;
@Test
@@ -56,7 +56,7 @@ public class PreparedStatementTest {
ConnectionFactory._transportFactory = _previousTransportFactory;
}
- class DummyPinotClientTransport implements PinotClientTransport {
+ static class DummyPinotClientTransport implements PinotClientTransport {
private String _lastQuery;
@Override
diff --git a/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/ResultSetGroupTest.java b/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/ResultSetGroupTest.java
index f31fe8a..900ec1e 100644
--- a/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/ResultSetGroupTest.java
+++ b/pinot-clients/pinot-java-client/src/test/java/org/apache/pinot/client/ResultSetGroupTest.java
@@ -32,7 +32,7 @@ import org.testng.annotations.Test;
*
*/
public class ResultSetGroupTest {
- private DummyJsonTransport _dummyJsonTransport = new DummyJsonTransport();
+ private final DummyJsonTransport _dummyJsonTransport = new DummyJsonTransport();
private PinotClientTransportFactory _previousTransportFactory = null;
@Test
@@ -55,11 +55,15 @@ public class ResultSetGroupTest {
Assert.assertEquals(resultSet.getColumnCount(), 79);
Assert.assertEquals(resultSet.getColumnName(0), "ActualElapsedTime");
Assert.assertEquals(resultSet.getColumnName(1), "AirTime");
+
+ // Verify the execution stats.
+ Assert.assertEquals(115545, resultSetGroup.getExecutionStats().getTotalDocs());
+ Assert.assertEquals(82, resultSetGroup.getExecutionStats().getTimeUsedMs());
+ Assert.assertEquals(24, resultSetGroup.getExecutionStats().getNumDocsScanned());
}
@Test
- public void testDeserializeAggregationResultSet()
- throws Exception {
+ public void testDeserializeAggregationResultSet() {
// Deserialize aggregation result
ResultSetGroup resultSetGroup = getResultSet("aggregation.json");
@@ -131,7 +135,7 @@ public class ResultSetGroupTest {
ConnectionFactory._transportFactory = _previousTransportFactory;
}
- class DummyJsonTransport implements PinotClientTransport {
+ static class DummyJsonTransport implements PinotClientTransport {
public String _resource;
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org