You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by su...@apache.org on 2018/11/17 00:11:04 UTC
[incubator-pinot] 01/01: [PINOT-7370] Track inverted index loaded
per query
This is an automated email from the ASF dual-hosted git repository.
sunithabeeram pushed a commit to branch trackIndexWork
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8f373f1026ac583bf5f9bd85060c3e213617abf9
Author: Sunitha Beeram <sb...@sbeeram-ld2.linkedin.biz>
AuthorDate: Fri Nov 16 16:10:39 2018 -0800
[PINOT-7370] Track inverted index loaded per query
---
.../main/java/com/linkedin/pinot/common/utils/DataTable.java | 1 +
.../linkedin/pinot/core/operator/CombineGroupByOperator.java | 1 +
.../com/linkedin/pinot/core/operator/CombineOperator.java | 1 +
.../com/linkedin/pinot/core/operator/DocIdSetOperator.java | 3 ++-
.../linkedin/pinot/core/operator/ExecutionStatistics.java | 12 ++++++++++--
.../pinot/core/operator/blocks/IntermediateResultsBlock.java | 6 ++++++
.../pinot/core/operator/docidsets/AndBlockDocIdSet.java | 9 +++++++++
.../pinot/core/operator/docidsets/BitmapDocIdSet.java | 12 +++++++++---
.../core/operator/docidsets/EmptyFilterBlockDocIdSet.java | 5 +++++
.../pinot/core/operator/docidsets/FilterBlockDocIdSet.java | 5 +++++
.../pinot/core/operator/docidsets/OrBlockDocIdSet.java | 9 +++++++++
.../core/operator/docidsets/ScanBasedMultiValueDocIdSet.java | 5 +++++
.../operator/docidsets/ScanBasedSingleValueDocIdSet.java | 5 +++++
.../pinot/core/operator/docidsets/SizeBasedDocIdSet.java | 5 +++++
.../pinot/core/operator/docidsets/SortedDocIdSet.java | 5 +++++
.../pinot/core/operator/docidsets/StarTreeDocIdSet.java | 6 ++++++
.../core/operator/query/AggregationGroupByOperator.java | 4 ++--
.../pinot/core/operator/query/AggregationOperator.java | 4 ++--
.../operator/query/DictionaryBasedAggregationOperator.java | 3 ++-
.../pinot/core/operator/query/EmptySelectionOperator.java | 2 +-
.../operator/query/MetadataBasedAggregationOperator.java | 3 ++-
.../pinot/core/operator/query/SelectionOnlyOperator.java | 4 ++--
.../pinot/core/operator/query/SelectionOrderByOperator.java | 4 ++--
.../linkedin/pinot/core/query/scheduler/QueryScheduler.java | 6 ++++--
.../pinot/operator/filter/FilterOperatorTestUtils.java | 5 +++++
25 files changed, 106 insertions(+), 19 deletions(-)
diff --git a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java
index caaa397..9c6a7e9 100644
--- a/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java
+++ b/pinot-common/src/main/java/com/linkedin/pinot/common/utils/DataTable.java
@@ -27,6 +27,7 @@ import javax.annotation.Nonnull;
public interface DataTable {
String EXCEPTION_METADATA_KEY = "Exception";
String NUM_DOCS_SCANNED_METADATA_KEY = "numDocsScanned";
+ String NUM_INDICES_LOADED_KEY = "numIndicesLoaded";
String NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY = "numEntriesScannedInFilter";
String NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY = "numEntriesScannedPostFilter";
String TOTAL_DOCS_METADATA_KEY = "totalDocs";
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java
index 023b7eb..f552269 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineGroupByOperator.java
@@ -190,6 +190,7 @@ public class CombineGroupByOperator extends BaseOperator<IntermediateResultsBloc
}
}
mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned());
+ mergedBlock.setNumIndicesLoaded(executionStatistics.getNumIndicesLoaded());
mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter());
mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter());
mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs());
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java
index 107a687..c2b81b7 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/CombineOperator.java
@@ -184,6 +184,7 @@ public class CombineOperator extends BaseOperator<IntermediateResultsBlock> {
}
}
mergedBlock.setNumDocsScanned(executionStatistics.getNumDocsScanned());
+ mergedBlock.setNumIndicesLoaded(executionStatistics.getNumIndicesLoaded());
mergedBlock.setNumEntriesScannedInFilter(executionStatistics.getNumEntriesScannedInFilter());
mergedBlock.setNumEntriesScannedPostFilter(executionStatistics.getNumEntriesScannedPostFilter());
mergedBlock.setNumTotalRawDocs(executionStatistics.getNumTotalRawDocs());
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java
index 03858a3..cbb88a1 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/DocIdSetOperator.java
@@ -88,6 +88,7 @@ public class DocIdSetOperator extends BaseOperator<DocIdSetBlock> {
@Override
public ExecutionStatistics getExecutionStatistics() {
- return new ExecutionStatistics(0L, _filterBlockDocIdSet.getNumEntriesScannedInFilter(), 0L, 0L);
+ return new ExecutionStatistics(0L, _filterBlockDocIdSet.getNumIndicesLoaded(),
+ _filterBlockDocIdSet.getNumEntriesScannedInFilter(), 0L, 0L);
}
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java
index 2a1d9eb..eb0674f 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/ExecutionStatistics.java
@@ -20,6 +20,7 @@ package com.linkedin.pinot.core.operator;
*/
public class ExecutionStatistics {
private long _numDocsScanned;
+ private long _numIndicesLoaded;
private long _numEntriesScannedInFilter;
private long _numEntriesScannedPostFilter;
private long _numTotalRawDocs;
@@ -27,9 +28,10 @@ public class ExecutionStatistics {
public ExecutionStatistics() {
}
- public ExecutionStatistics(long numDocsScanned, long numEntriesScannedInFilter, long numEntriesScannedPostFilter,
- long numTotalRawDocs) {
+ public ExecutionStatistics(long numDocsScanned, long numIndicesLoaded, long numEntriesScannedInFilter,
+ long numEntriesScannedPostFilter, long numTotalRawDocs) {
_numDocsScanned = numDocsScanned;
+ _numIndicesLoaded = numIndicesLoaded;
_numEntriesScannedInFilter = numEntriesScannedInFilter;
_numEntriesScannedPostFilter = numEntriesScannedPostFilter;
_numTotalRawDocs = numTotalRawDocs;
@@ -39,6 +41,10 @@ public class ExecutionStatistics {
return _numDocsScanned;
}
+ public long getNumIndicesLoaded() {
+ return _numIndicesLoaded;
+ }
+
public long getNumEntriesScannedInFilter() {
return _numEntriesScannedInFilter;
}
@@ -58,6 +64,7 @@ public class ExecutionStatistics {
*/
public void merge(ExecutionStatistics executionStatisticsToMerge) {
_numDocsScanned += executionStatisticsToMerge._numDocsScanned;
+ _numIndicesLoaded += executionStatisticsToMerge._numIndicesLoaded;
_numEntriesScannedInFilter += executionStatisticsToMerge._numEntriesScannedInFilter;
_numEntriesScannedPostFilter += executionStatisticsToMerge._numEntriesScannedPostFilter;
_numTotalRawDocs += executionStatisticsToMerge._numTotalRawDocs;
@@ -67,6 +74,7 @@ public class ExecutionStatistics {
public String toString() {
return "Execution Statistics:"
+ "\n numDocsScanned: " + _numDocsScanned
+ + "\n numIndicesLoaded: " + _numIndicesLoaded
+ "\n numEntriesScannedInFilter: " + _numEntriesScannedInFilter
+ "\n numEntriesScannedPostFilter: " + _numEntriesScannedPostFilter
+ "\n numTotalRawDocs: " + _numTotalRawDocs;
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java
index 80da626..06e50bc 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/blocks/IntermediateResultsBlock.java
@@ -50,6 +50,7 @@ public class IntermediateResultsBlock implements Block {
private List<Map<String, Object>> _combinedAggregationGroupByResult;
private List<ProcessingException> _processingExceptions;
private long _numDocsScanned;
+ private long _numIndicesLoaded;
private long _numEntriesScannedInFilter;
private long _numEntriesScannedPostFilter;
private long _numTotalRawDocs;
@@ -165,6 +166,10 @@ public class IntermediateResultsBlock implements Block {
_numDocsScanned = numDocsScanned;
}
+ public void setNumIndicesLoaded(long numIndicesLoaded) {
+ _numIndicesLoaded = numIndicesLoaded;
+ }
+
public void setNumEntriesScannedInFilter(long numEntriesScannedInFilter) {
_numEntriesScannedInFilter = numEntriesScannedInFilter;
}
@@ -275,6 +280,7 @@ public class IntermediateResultsBlock implements Block {
private DataTable attachMetadataToDataTable(DataTable dataTable) {
dataTable.getMetadata().put(DataTable.NUM_DOCS_SCANNED_METADATA_KEY, String.valueOf(_numDocsScanned));
+ dataTable.getMetadata().put(DataTable.NUM_INDICES_LOADED_KEY, String.valueOf(_numIndicesLoaded));
dataTable.getMetadata()
.put(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY, String.valueOf(_numEntriesScannedInFilter));
dataTable.getMetadata()
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java
index f6cefd3..85adc6e 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/AndBlockDocIdSet.java
@@ -259,4 +259,13 @@ public final class AndBlockDocIdSet implements FilterBlockDocIdSet {
}
return numEntriesScannedInFilter;
}
+
+ @Override
+ public long getNumIndicesLoaded() {
+ long loaded = 0;
+ for (FilterBlockDocIdSet docIdSet : blockDocIdSets) {
+ loaded += docIdSet.getNumIndicesLoaded();
+ }
+ return loaded;
+ }
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java
index ba56f47..556eab3 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/BitmapDocIdSet.java
@@ -23,19 +23,20 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
public class BitmapDocIdSet implements FilterBlockDocIdSet {
private final ImmutableRoaringBitmap _bitmap;
+ private final long _bitMapCount;
private int _startDocId;
// Inclusive
private int _endDocId;
public BitmapDocIdSet(ImmutableRoaringBitmap[] bitmaps, int startDocId, int endDocId, boolean exclusive) {
- int numBitmaps = bitmaps.length;
- if (numBitmaps > 1) {
+ _bitMapCount = bitmaps.length;
+ if (_bitMapCount > 1) {
MutableRoaringBitmap orBitmap = MutableRoaringBitmap.or(bitmaps);
if (exclusive) {
orBitmap.flip(startDocId, endDocId + 1);
}
_bitmap = orBitmap;
- } else if (numBitmaps == 1) {
+ } else if (_bitMapCount == 1) {
if (exclusive) {
// NOTE: cannot use ImmutableRoaringBitmap.flip() because the library has a bug in that method
// TODO: the bug has been fixed in the latest version of ImmutableRoaringBitmap, update the version
@@ -83,6 +84,11 @@ public class BitmapDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ return _bitMapCount;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
BitmapDocIdIterator bitmapDocIdIterator = new BitmapDocIdIterator(_bitmap.getIntIterator());
bitmapDocIdIterator.setStartDocId(_startDocId);
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java
index af02c83..87c64c2 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/EmptyFilterBlockDocIdSet.java
@@ -56,6 +56,11 @@ public final class EmptyFilterBlockDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ return 0L;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
return EmptyBlockDocIdIterator.getInstance();
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java
index 7dcf864..22721c3 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/FilterBlockDocIdSet.java
@@ -68,4 +68,9 @@ public interface FilterBlockDocIdSet extends BlockDocIdSet {
* Returns the number of entries scanned in filtering phase.
*/
long getNumEntriesScannedInFilter();
+
+ /**
+ * Returns the number of indices loaded in filtering phase.
+ */
+ long getNumIndicesLoaded();
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java
index bd18cb4..17de4bf 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/OrBlockDocIdSet.java
@@ -72,6 +72,15 @@ public final class OrBlockDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ long loaded = 0;
+ for (FilterBlockDocIdSet docIdSet : _docIdSets) {
+ loaded += docIdSet.getNumIndicesLoaded();
+ }
+ return loaded;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
boolean useBitmapOr = false;
for (BlockDocIdSet docIdSet : _docIdSets) {
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java
index 2fdd038..c5162b8 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedMultiValueDocIdSet.java
@@ -72,6 +72,11 @@ public class ScanBasedMultiValueDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ return 0L;
+ }
+
+ @Override
public ScanBasedDocIdIterator iterator() {
return blockValSetBlockDocIdIterator;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java
index 6594586..ccb5d55 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/ScanBasedSingleValueDocIdSet.java
@@ -73,6 +73,11 @@ public class ScanBasedSingleValueDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ return 0L;
+ }
+
+ @Override
public ScanBasedDocIdIterator iterator() {
return blockValSetBlockDocIdIterator;
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java
index bbd6626..d5d0098 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SizeBasedDocIdSet.java
@@ -50,6 +50,11 @@ public final class SizeBasedDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ return 0L;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
return new SizeBasedDocIdIterator(_maxDocId);
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java
index 13c8b20..8514029 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/SortedDocIdSet.java
@@ -78,6 +78,11 @@ public class SortedDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ return 0L;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
if (pairs == null || pairs.isEmpty()) {
return EmptyBlockDocIdIterator.getInstance();
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java
index 1b14c5c..0c55688 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docidsets/StarTreeDocIdSet.java
@@ -89,6 +89,12 @@ public class StarTreeDocIdSet implements FilterBlockDocIdSet {
}
@Override
+ public long getNumIndicesLoaded() {
+ // TODO: this must be fixed
+ return 0L;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
return new RangeBasedDocIdIterator(minDocId, maxDocId);
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java
index a5a3b47..2dff034 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationGroupByOperator.java
@@ -83,8 +83,8 @@ public class AggregationGroupByOperator extends BaseOperator<IntermediateResults
long numEntriesScannedInFilter = _transformOperator.getExecutionStatistics().getNumEntriesScannedInFilter();
long numEntriesScannedPostFilter = numDocsScanned * _transformOperator.getNumColumnsProjected();
_executionStatistics =
- new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter,
- _numTotalRawDocs);
+ new ExecutionStatistics(numDocsScanned,_transformOperator.getExecutionStatistics().getNumIndicesLoaded(),
+ numEntriesScannedInFilter, numEntriesScannedPostFilter, _numTotalRawDocs);
// Build intermediate result block based on aggregation group-by result from the executor
return new IntermediateResultsBlock(_functionContexts, groupByResult);
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java
index 776f122..e5d3910 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/AggregationOperator.java
@@ -71,8 +71,8 @@ public class AggregationOperator extends BaseOperator<IntermediateResultsBlock>
long numEntriesScannedInFilter = _transformOperator.getExecutionStatistics().getNumEntriesScannedInFilter();
long numEntriesScannedPostFilter = numDocsScanned * _transformOperator.getNumColumnsProjected();
_executionStatistics =
- new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter,
- _numTotalRawDocs);
+ new ExecutionStatistics(numDocsScanned, _transformOperator.getExecutionStatistics().getNumIndicesLoaded(),
+ numEntriesScannedInFilter, numEntriesScannedPostFilter, _numTotalRawDocs);
// Build intermediate result block based on aggregation result from the executor
return new IntermediateResultsBlock(_functionContexts, aggregationResult, false);
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java
index 749ff7e..840a6a7 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/DictionaryBasedAggregationOperator.java
@@ -95,7 +95,8 @@ public class DictionaryBasedAggregationOperator extends BaseOperator<Intermediat
// Create execution statistics. Set numDocsScanned to totalRawDocs for backward compatibility.
_executionStatistics =
- new ExecutionStatistics(_totalRawDocs, 0/* numEntriesScannedInFilter */, 0/* numEntriesScannedPostFilter */,
+ new ExecutionStatistics(_totalRawDocs, 0/*numIndicesLoaded*/,
+ 0/* numEntriesScannedInFilter */, 0/* numEntriesScannedPostFilter */,
_totalRawDocs);
// Build intermediate result block based on aggregation result from the executor.
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java
index 025bc79..c421547 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/EmptySelectionOperator.java
@@ -42,7 +42,7 @@ public class EmptySelectionOperator extends BaseOperator<IntermediateResultsBloc
List<String> selectionColumns =
SelectionOperatorUtils.getSelectionColumns(selection.getSelectionColumns(), indexSegment);
_dataSchema = SelectionOperatorUtils.extractDataSchema(null, selectionColumns, indexSegment);
- _executionStatistics = new ExecutionStatistics(0L, 0L, 0L, indexSegment.getSegmentMetadata().getTotalRawDocs());
+ _executionStatistics = new ExecutionStatistics(0L, 0L, 0L, 0L, indexSegment.getSegmentMetadata().getTotalRawDocs());
}
@Override
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java
index ce9087b..b1650f1 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/MetadataBasedAggregationOperator.java
@@ -70,7 +70,8 @@ public class MetadataBasedAggregationOperator extends BaseOperator<IntermediateR
// Create execution statistics. Set numDocsScanned to totalRawDocs for backward compatibility.
_executionStatistics =
- new ExecutionStatistics(totalRawDocs, 0/*numEntriesScannedInFilter*/, 0/*numEntriesScannedPostFilter*/,
+ new ExecutionStatistics(totalRawDocs, 0/*numIndicesLoaded*/,
+ 0/*numEntriesScannedInFilter*/, 0/*numEntriesScannedPostFilter*/,
totalRawDocs);
// Build intermediate result block based on aggregation result from the executor.
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java
index 5f2051f..8ebb313 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOnlyOperator.java
@@ -88,8 +88,8 @@ public class SelectionOnlyOperator extends BaseOperator<IntermediateResultsBlock
long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumColumnsProjected();
long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
_executionStatistics =
- new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter,
- numTotalRawDocs);
+ new ExecutionStatistics(numDocsScanned, _projectionOperator.getExecutionStatistics().getNumIndicesLoaded(),
+ numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs);
return new IntermediateResultsBlock(_dataSchema, _rowEvents);
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java
index 2e5f35b..84dfe35 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/query/SelectionOrderByOperator.java
@@ -98,8 +98,8 @@ public class SelectionOrderByOperator extends BaseOperator<IntermediateResultsBl
long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumColumnsProjected();
long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
_executionStatistics =
- new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter,
- numTotalRawDocs);
+ new ExecutionStatistics(numDocsScanned, _projectionOperator.getExecutionStatistics().getNumIndicesLoaded(),
+ numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs);
return new IntermediateResultsBlock(_selectionOperatorService.getDataSchema(), _selectionOperatorService.getRows());
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java b/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java
index 1edb97c..411d218 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/query/scheduler/QueryScheduler.java
@@ -143,6 +143,8 @@ public abstract class QueryScheduler {
String tableNameWithType = queryRequest.getTableNameWithType();
long numDocsScanned =
Long.parseLong(dataTableMetadata.getOrDefault(DataTable.NUM_DOCS_SCANNED_METADATA_KEY, INVALID_NUM_SCANNED));
+ long numIndicesLoaded =
+ Long.parseLong(dataTableMetadata.getOrDefault(DataTable.NUM_INDICES_LOADED_KEY, "0"));
long numEntriesScannedInFilter = Long.parseLong(
dataTableMetadata.getOrDefault(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY, INVALID_NUM_SCANNED));
long numEntriesScannedPostFilter = Long.parseLong(
@@ -161,11 +163,11 @@ public abstract class QueryScheduler {
TimerContext timerContext = queryRequest.getTimerContext();
LOGGER.info(
- "Processed requestId={},table={},reqSegments={},prunedToSegmentCount={},totalExecMs={},totalTimeMs={},broker={},numDocsScanned={},scanInFilter={},scanPostFilter={},sched={}",
+ "Processed requestId={},table={},reqSegments={},prunedToSegmentCount={},totalExecMs={},totalTimeMs={},broker={},numDocsScanned={},numIndicesLoaded={},scanInFilter={},scanPostFilter={},sched={}",
requestId, tableNameWithType, queryRequest.getSegmentsToQuery().size(),
queryRequest.getSegmentCountAfterPruning(), timerContext.getPhaseDurationMs(ServerQueryPhase.QUERY_PROCESSING),
timerContext.getPhaseDurationMs(ServerQueryPhase.TOTAL_QUERY_TIME), queryRequest.getBrokerId(), numDocsScanned,
- numEntriesScannedInFilter, numEntriesScannedPostFilter, name());
+ numIndicesLoaded, numEntriesScannedInFilter, numEntriesScannedPostFilter, name());
serverMetrics.addMeteredTableValue(tableNameWithType, ServerMeter.NUM_SEGMENTS_SEARCHED,
queryRequest.getSegmentCountAfterPruning());
diff --git a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java b/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java
index 58fb7db..76cd00c 100644
--- a/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java
+++ b/pinot-core/src/test/java/com/linkedin/pinot/operator/filter/FilterOperatorTestUtils.java
@@ -65,6 +65,11 @@ public class FilterOperatorTestUtils {
}
@Override
+ public long getNumIndicesLoaded() {
+ return 0;
+ }
+
+ @Override
public BlockDocIdIterator iterator() {
return new ArrayBasedDocIdIterator(docIds, docIds.length);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org