You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2018/11/20 18:23:42 UTC
[incubator-pinot] 01/01: Adding execution stats for
numSegmentsQueried/Processed/Matched
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch execution-stats
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit de9d74cdf7860d0eef5f005aac6cc20a185bd08e
Author: kishoreg <ki...@apache.org>
AuthorDate: Mon Nov 19 18:28:53 2018 -0800
Adding execution stats for numSegmentsQueried/Processed/Matched
---
.../response/broker/BrokerResponseNative.java | 25 +++++++++++++++++++++-
.../com/linkedin/pinot/common/utils/DataTable.java | 3 +++
.../core/operator/CombineGroupByOperator.java | 2 ++
.../pinot/core/operator/CombineOperator.java | 2 ++
.../pinot/core/operator/ExecutionStatistics.java | 20 +++++++++++++++--
.../operator/blocks/IntermediateResultsBlock.java | 21 ++++++++++++++++++
.../core/query/reduce/BrokerReduceService.java | 13 +++++++++++
7 files changed, 83 insertions(+), 3 deletions(-)
diff --git a/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java b/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java
index e4a251c..43b0ade 100644
--- a/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java
+++ b/pinot-common/src/main/java/com/linkedin/pinot/common/response/broker/BrokerResponseNative.java
@@ -40,7 +40,7 @@ import org.json.JSONObject;
* Supports serialization via JSON.
*/
@JsonPropertyOrder({"selectionResults", "aggregationResults", "exceptions", "numServersQueried", "numServersResponded",
- "numDocsScanned", "numEntriesScannedInFilter", "numEntriesScannedPostFilter", "totalDocs", "numGroupsLimitReached",
+ "numDocsScanned", "numEntriesScannedInFilter", "numEntriesScannedPostFilter", "numSegmentsProcessed", "numSegmentsWithNoMatch", "totalDocs", "numGroupsLimitReached",
"timeUsedMs", "segmentStatistics", "traceInfo"})
public class BrokerResponseNative implements BrokerResponse {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -54,6 +54,9 @@ public class BrokerResponseNative implements BrokerResponse {
private long _numDocsScanned = 0L;
private long _numEntriesScannedInFilter = 0L;
private long _numEntriesScannedPostFilter = 0L;
+ private long _numSegmentsProcessed = 0L;
+ private long _numSegmentsWithNoMatch = 0L;
+
private long _totalDocs = 0L;
private boolean _numGroupsLimitReached = false;
private long _timeUsedMs = 0L;
@@ -173,6 +176,26 @@ public class BrokerResponseNative implements BrokerResponse {
_numEntriesScannedPostFilter = numEntriesScannedPostFilter;
}
+ @JsonProperty("numSegmentsProcessed")
+ public long getNumSegmentsProcessed() {
+ return _numSegmentsProcessed;
+ }
+
+ @JsonProperty("numSegmentsProcessed")
+ public void setNumSegmentsProcessed(long numSegmentsProcessed) {
+ _numSegmentsProcessed = numSegmentsProcessed;
+ }
+
+ @JsonProperty("numSegmentsWithNoMatch")
+ public long getNumSegmentsWithNoMatch() {
+ return _numSegmentsWithNoMatch;
+ }
+
+ @JsonProperty("numSegmentsWithNoMatch")
+ public void setNumSegmentsWithNoMatch(long numSegmentsWithNoMatch) {
+ _numSegmentsWithNoMatch = numSegmentsWithNoMatch;
+ }
+
@JsonProperty("totalDocs")
@Override
public long getTotalDocs() {
diff --git a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java
index caaa397..09066bf 100644
--- a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java
+++ b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java
@@ -29,6 +29,9 @@ public interface DataTable {
String NUM_DOCS_SCANNED_METADATA_KEY = "numDocsScanned";
String NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY = "numEntriesScannedInFilter";
String NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY = "numEntriesScannedPostFilter";
+ String NUM_SEGMENTS_QUERIED = "numSegmentsQueried";
+ String NUM_SEGMENTS_PROCESSED = "numSegmentsProcessed";
+ String NUM_SEGMENTS_WITH_NO_MATCH = "numSegmentsWithNoMatch";
String TOTAL_DOCS_METADATA_KEY = "totalDocs";
String NUM_GROUPS_LIMIT_REACHED_KEY = "numGroupsLimitReached";
String TIME_USED_MS_METADATA_KEY = "timeUsedMs";
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java
index 023b7eb..e4593f8 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java
@@ -192,6 +192,8 @@ public class CombineGroupByOperator extends BaseOperator<IntermediateResultsBloc
mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned());
mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter());
mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter());
+ mergedBlock.setNumSegmentsProcessed(executionStatistics.getNumSegmentsProcessed());
+ mergedBlock.setNumSegmentsWithNoMatch(executionStatistics.getNumSegmentsWithNoMatch());
mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs());
// NOTE: numGroups might go slightly over numGroupsLimit because the comparison is not atomic
if (numGroups.get() >= _numGroupsLimit) {
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java
index 107a687..c7148ec 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java
@@ -187,6 +187,8 @@ public class CombineOperator extends BaseOperator<IntermediateResultsBlock> {
mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter());
mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter());
mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs());
+ mergedBlock.setNumSegmentsProcessed(executionStatistics.getNumSegmentsProcessed());
+ mergedBlock.setNumSegmentsWithNoMatch(executionStatistics.getNumSegmentsWithNoMatch());
return mergedBlock;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java
index 2a1d9eb..0a2919d 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java
@@ -23,7 +23,9 @@ public class ExecutionStatistics {
private long _numEntriesScannedInFilter;
private long _numEntriesScannedPostFilter;
private long _numTotalRawDocs;
-
+ private long _numSegmentsProcessed;
+ private long _numSegmentsWithNoMatch;
+
public ExecutionStatistics() {
}
@@ -33,6 +35,8 @@ public class ExecutionStatistics {
_numEntriesScannedInFilter = numEntriesScannedInFilter;
_numEntriesScannedPostFilter = numEntriesScannedPostFilter;
_numTotalRawDocs = numTotalRawDocs;
+ _numSegmentsProcessed = 1;
+ _numSegmentsWithNoMatch = (numDocsScanned == 0) ? 1 : 0;
}
public long getNumDocsScanned() {
@@ -51,6 +55,14 @@ public class ExecutionStatistics {
return _numTotalRawDocs;
}
+ public long getNumSegmentsProcessed() {
+ return _numSegmentsProcessed;
+ }
+
+ public long getNumSegmentsWithNoMatch() {
+ return _numSegmentsWithNoMatch;
+ }
+
/**
* Merge another execution statistics into the current one.
*
@@ -61,6 +73,8 @@ public class ExecutionStatistics {
_numEntriesScannedInFilter += executionStatisticsToMerge._numEntriesScannedInFilter;
_numEntriesScannedPostFilter += executionStatisticsToMerge._numEntriesScannedPostFilter;
_numTotalRawDocs += executionStatisticsToMerge._numTotalRawDocs;
+ _numSegmentsProcessed += executionStatisticsToMerge._numSegmentsProcessed;
+ _numSegmentsWithNoMatch += executionStatisticsToMerge._numSegmentsWithNoMatch;
}
@Override
@@ -69,6 +83,8 @@ public class ExecutionStatistics {
+ "\n numDocsScanned: " + _numDocsScanned
+ "\n numEntriesScannedInFilter: " + _numEntriesScannedInFilter
+ "\n numEntriesScannedPostFilter: " + _numEntriesScannedPostFilter
- + "\n numTotalRawDocs: " + _numTotalRawDocs;
+ + "\n numTotalRawDocs: " + _numTotalRawDocs
+ + "\n numSegmentsProcessed: " + _numSegmentsProcessed
+ + "\n numSegmentsWithNoMatch: " + _numSegmentsWithNoMatch;
}
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java
index 80da626..aebbf5e 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java
@@ -53,6 +53,8 @@ public class IntermediateResultsBlock implements Block {
private long _numEntriesScannedInFilter;
private long _numEntriesScannedPostFilter;
private long _numTotalRawDocs;
+ private long _numSegmentsProcessed;
+ private long _numSegmentsWithNoMatch;
private boolean _numGroupsLimitReached;
/**
@@ -172,6 +174,22 @@ public class IntermediateResultsBlock implements Block {
public void setNumEntriesScannedPostFilter(long numEntriesScannedPostFilter) {
_numEntriesScannedPostFilter = numEntriesScannedPostFilter;
}
+
+ public long getNumSegmentsProcessed() {
+ return _numSegmentsProcessed;
+ }
+
+ public void setNumSegmentsProcessed(long numSegmentsProcessed) {
+ _numSegmentsProcessed = numSegmentsProcessed;
+ }
+
+ public long getNumSegmentsWithNoMatch() {
+ return _numSegmentsWithNoMatch;
+ }
+
+ public void setNumSegmentsWithNoMatch(long numSegmentsWithNoMatch) {
+ _numSegmentsWithNoMatch = numSegmentsWithNoMatch;
+ }
public void setNumTotalRawDocs(long numTotalRawDocs) {
_numTotalRawDocs = numTotalRawDocs;
@@ -279,6 +297,9 @@ public class IntermediateResultsBlock implements Block {
.put(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY, String.valueOf(_numEntriesScannedInFilter));
dataTable.getMetadata()
.put(DataTable.NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY, String.valueOf(_numEntriesScannedPostFilter));
+ dataTable.getMetadata().put(DataTable.NUM_SEGMENTS_PROCESSED, String.valueOf(_numSegmentsProcessed));
+ dataTable.getMetadata().put(DataTable.NUM_SEGMENTS_WITH_NO_MATCH, String.valueOf(_numSegmentsWithNoMatch));
+
dataTable.getMetadata().put(DataTable.TOTAL_DOCS_METADATA_KEY, String.valueOf(_numTotalRawDocs));
if (_numGroupsLimitReached) {
dataTable.getMetadata().put(DataTable.NUM_GROUPS_LIMIT_REACHED_KEY, "true");
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java b/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java
index 1c43b40..483ae9d 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/query/reduce/BrokerReduceService.java
@@ -76,6 +76,8 @@ public class BrokerReduceService implements ReduceService<BrokerResponseNative>
long numDocsScanned = 0L;
long numEntriesScannedInFilter = 0L;
long numEntriesScannedPostFilter = 0L;
+ long numSegmentsProcessed = 0L;
+ long numSegmentsWithNoMatch = 0L;
long numTotalRawDocs = 0L;
boolean numGroupsLimitReached = false;
@@ -116,6 +118,15 @@ public class BrokerReduceService implements ReduceService<BrokerResponseNative>
if (numEntriesScannedPostFilterString != null) {
numEntriesScannedPostFilter += Long.parseLong(numEntriesScannedPostFilterString);
}
+ String numSegmentsProcessedString = metadata.get(DataTable.NUM_SEGMENTS_PROCESSED);
+ if (numSegmentsProcessedString != null) {
+ numSegmentsProcessed += Long.parseLong(numSegmentsProcessedString);
+ }
+ String numSegmentsWithNoMatchString = metadata.get(DataTable.NUM_SEGMENTS_WITH_NO_MATCH);
+ if (numSegmentsProcessedString != null) {
+ numSegmentsWithNoMatch += Long.parseLong(numSegmentsWithNoMatchString);
+ }
+
String numTotalRawDocsString = metadata.get(DataTable.TOTAL_DOCS_METADATA_KEY);
if (numTotalRawDocsString != null) {
numTotalRawDocs += Long.parseLong(numTotalRawDocsString);
@@ -143,6 +154,8 @@ public class BrokerReduceService implements ReduceService<BrokerResponseNative>
brokerResponseNative.setNumDocsScanned(numDocsScanned);
brokerResponseNative.setNumEntriesScannedInFilter(numEntriesScannedInFilter);
brokerResponseNative.setNumEntriesScannedPostFilter(numEntriesScannedPostFilter);
+ brokerResponseNative.setNumSegmentsProcessed(numSegmentsProcessed);
+ brokerResponseNative.setNumSegmentsWithNoMatch(numSegmentsWithNoMatch);
brokerResponseNative.setTotalDocs(numTotalRawDocs);
brokerResponseNative.setNumGroupsLimitReached(numGroupsLimitReached);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org