You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2017/06/30 16:49:01 UTC
lucene-solr:jira/solr-10990: SOLR-10990: QueryComponent.process
breakup (for readability)
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-10990 [created] e8a704a82
SOLR-10990: QueryComponent.process breakup (for readability)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e8a704a8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e8a704a8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e8a704a8
Branch: refs/heads/jira/solr-10990
Commit: e8a704a824ea5b38eea34f664857105665bd359d
Parents: 15118d4
Author: Christine Poerschke <cp...@apache.org>
Authored: Fri Jun 30 17:40:06 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Fri Jun 30 17:47:33 2017 +0100
----------------------------------------------------------------------
.../solr/handler/component/QueryComponent.java | 467 +++++++++++--------
1 file changed, 272 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8a704a8/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 1d88adc..417d583 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -306,12 +306,12 @@ public class QueryComponent extends SearchComponent
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
- SolrIndexSearcher searcher = req.getSearcher();
StatsCache statsCache = req.getCore().getStatsCache();
int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
+ SolrIndexSearcher searcher = req.getSearcher();
statsCache.returnLocalStats(rb, searcher);
return;
}
@@ -321,50 +321,11 @@ public class QueryComponent extends SearchComponent
statsCache.receiveGlobalStats(req);
}
- SolrQueryResponse rsp = rb.rsp;
- IndexSchema schema = searcher.getSchema();
-
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
- String ids = params.get(ShardParams.IDS);
- if (ids != null) {
- SchemaField idField = schema.getUniqueKeyField();
- List<String> idArr = StrUtils.splitSmart(ids, ",", true);
- int[] luceneIds = new int[idArr.size()];
- int docs = 0;
- if (idField.getType().isPointField()) {
- for (int i=0; i<idArr.size(); i++) {
- int id = searcher.search(
- idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
- if (id >= 0) {
- luceneIds[docs++] = id;
- }
- }
- } else {
- for (int i=0; i<idArr.size(); i++) {
- int id = searcher.getFirstMatch(
- new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
- if (id >= 0)
- luceneIds[docs++] = id;
- }
- }
-
- DocListAndSet res = new DocListAndSet();
- res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
- if (rb.isNeedDocSet()) {
- // TODO: create a cache for this!
- List<Query> queries = new ArrayList<>();
- queries.add(rb.getQuery());
- List<Query> filters = rb.getFilters();
- if (filters != null) queries.addAll(filters);
- res.docSet = searcher.getDocSet(queries);
- }
- rb.setResults(res);
-
- ResultContext ctx = new BasicResultContext(rb);
- rsp.addResponse(ctx);
+ if (doProcessSearchByIds(rb)) {
return;
}
@@ -395,145 +356,15 @@ public class QueryComponent extends SearchComponent
if (groupingSpec != null) {
cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
try {
- boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
- CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
- .setQueryCommand(cmd)
- .setNeedDocSet(false) // Order matters here
- .setIncludeHitCount(true)
- .setSearcher(searcher);
-
- for (String field : groupingSpec.getFields()) {
- topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
- .setField(schema.getField(field))
- .setGroupSort(groupingSpec.getGroupSort())
- .setTopNGroups(cmd.getOffset() + cmd.getLen())
- .setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
- .build()
- );
- }
-
- CommandHandler commandHandler = topsGroupsActionBuilder.build();
- commandHandler.execute();
- SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
- rsp.add("firstPhase", commandHandler.processResult(result, serializer));
- rsp.add("totalHitCount", commandHandler.getTotalHitCount());
- rb.setResult(result);
+ doProcessGroupedDistributedSearchFirstPhase(rb, result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
- CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
- .setQueryCommand(cmd)
- .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
- .setSearcher(searcher);
-
- int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
- docsToCollect = Math.max(docsToCollect, 1);
-
- for (String field : groupingSpec.getFields()) {
- SchemaField schemaField = schema.getField(field);
- String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
- if (topGroupsParam == null) {
- topGroupsParam = new String[0];
- }
-
- List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
- for (String topGroup : topGroupsParam) {
- SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
- if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
- BytesRefBuilder builder = new BytesRefBuilder();
- schemaField.getType().readableToIndexed(topGroup, builder);
- searchGroup.groupValue = builder.get();
- }
- topGroups.add(searchGroup);
- }
-
- secondPhaseBuilder.addCommandField(
- new TopGroupsFieldCommand.Builder()
- .setField(schemaField)
- .setGroupSort(groupingSpec.getGroupSort())
- .setSortWithinGroup(groupingSpec.getSortWithinGroup())
- .setFirstPhaseGroups(topGroups)
- .setMaxDocPerGroup(docsToCollect)
- .setNeedScores(needScores)
- .setNeedMaxScore(needScores)
- .build()
- );
- }
-
- for (String query : groupingSpec.getQueries()) {
- secondPhaseBuilder.addCommandField(new Builder()
- .setDocsToCollect(docsToCollect)
- .setSort(groupingSpec.getGroupSort())
- .setQuery(query, rb.req)
- .setDocSet(searcher)
- .build()
- );
- }
-
- CommandHandler commandHandler = secondPhaseBuilder.build();
- commandHandler.execute();
- TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
- rsp.add("secondPhase", commandHandler.processResult(result, serializer));
- rb.setResult(result);
+ doProcessGroupedDistributedSearchSecondPhase(rb, result);
return;
}
- int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
- boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
- Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
- Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
- int limitDefault = cmd.getLen(); // this is normally from "rows"
- Grouping grouping =
- new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
- grouping.setGroupSort(groupingSpec.getGroupSort())
- .setWithinGroupSort(groupingSpec.getSortWithinGroup())
- .setDefaultFormat(groupingSpec.getResponseFormat())
- .setLimitDefault(limitDefault)
- .setDefaultTotalCount(defaultTotalCount)
- .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
- .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
- .setGetGroupedDocSet(groupingSpec.isTruncateGroups());
-
- if (groupingSpec.getFields() != null) {
- for (String field : groupingSpec.getFields()) {
- grouping.addFieldCommand(field, rb.req);
- }
- }
-
- if (groupingSpec.getFunctions() != null) {
- for (String groupByStr : groupingSpec.getFunctions()) {
- grouping.addFunctionCommand(groupByStr, rb.req);
- }
- }
-
- if (groupingSpec.getQueries() != null) {
- for (String groupByStr : groupingSpec.getQueries()) {
- grouping.addQueryCommand(groupByStr, rb.req);
- }
- }
-
- if( rb.isNeedDocList() || rb.isDebug() ){
- // we need a single list of the returned docs
- cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
- }
-
- grouping.execute();
- if (grouping.isSignalCacheWarning()) {
- rsp.add(
- "cacheWarning",
- String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
- );
- }
- rb.setResult(result);
-
- if (grouping.mainResult != null) {
- ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
- rsp.addResponse(ctx);
- rsp.getToLog().add("hits", grouping.mainResult.matches());
- } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
- rsp.add("grouped", result.groupedResults);
- rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
- }
+ doProcessGroupedSearch(rb, result);
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
@@ -541,27 +372,7 @@ public class QueryComponent extends SearchComponent
}
// normal search result
- searcher.search(result, cmd);
- rb.setResult(result);
-
- ResultContext ctx = new BasicResultContext(rb);
- rsp.addResponse(ctx);
- rsp.getToLog().add("hits", rb.getResults().docList.matches());
-
- if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
- if (null != rb.getNextCursorMark()) {
- rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
- rb.getNextCursorMark().getSerializedTotem());
- }
- }
-
- if(rb.mergeFieldHandler != null) {
- rb.mergeFieldHandler.handleMergeFields(rb, searcher);
- } else {
- doFieldSortValues(rb, searcher);
- }
-
- doPrefetch(rb);
+ doProcessUngroupedSearch(rb, result);
}
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
@@ -1385,6 +1196,272 @@ public class QueryComponent extends SearchComponent
return Category.QUERY;
}
+ private boolean doProcessSearchByIds(ResponseBuilder rb) throws IOException {
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrParams params = req.getParams();
+
+ String ids = params.get(ShardParams.IDS);
+ if (ids == null) {
+ return false;
+ }
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
+ SchemaField idField = schema.getUniqueKeyField();
+ List<String> idArr = StrUtils.splitSmart(ids, ",", true);
+ int[] luceneIds = new int[idArr.size()];
+ int docs = 0;
+ if (idField.getType().isPointField()) {
+ for (int i=0; i<idArr.size(); i++) {
+ int id = searcher.search(
+ idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
+ if (id >= 0) {
+ luceneIds[docs++] = id;
+ }
+ }
+ } else {
+ for (int i=0; i<idArr.size(); i++) {
+ int id = searcher.getFirstMatch(
+ new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
+ if (id >= 0)
+ luceneIds[docs++] = id;
+ }
+ }
+
+ DocListAndSet res = new DocListAndSet();
+ res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
+ if (rb.isNeedDocSet()) {
+ // TODO: create a cache for this!
+ List<Query> queries = new ArrayList<>();
+ queries.add(rb.getQuery());
+ List<Query> filters = rb.getFilters();
+ if (filters != null) queries.addAll(filters);
+ res.docSet = searcher.getDocSet(queries);
+ }
+ rb.setResults(res);
+
+ ResultContext ctx = new BasicResultContext(rb);
+ rsp.addResponse(ctx);
+ return true;
+ }
+
+ private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryResult result) throws IOException {
+
+ GroupingSpecification groupingSpec = rb.getGroupingSpec();
+ assert null != groupingSpec : "GroupingSpecification is null";
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
+
+ QueryCommand cmd = rb.getQueryCommand();
+
+ CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
+ .setQueryCommand(cmd)
+ .setNeedDocSet(false) // Order matters here
+ .setIncludeHitCount(true)
+ .setSearcher(searcher);
+
+ for (String field : groupingSpec.getFields()) {
+ topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
+ .setField(schema.getField(field))
+ .setGroupSort(groupingSpec.getGroupSort())
+ .setTopNGroups(cmd.getOffset() + cmd.getLen())
+ .setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
+ .build()
+ );
+ }
+
+ CommandHandler commandHandler = topsGroupsActionBuilder.build();
+ commandHandler.execute();
+ SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
+
+ rsp.add("firstPhase", commandHandler.processResult(result, serializer));
+ rsp.add("totalHitCount", commandHandler.getTotalHitCount());
+ rb.setResult(result);
+ }
+
+ private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
+
+ GroupingSpecification groupingSpec = rb.getGroupingSpec();
+ assert null != groupingSpec : "GroupingSpecification is null";
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrParams params = req.getParams();
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
+
+ QueryCommand cmd = rb.getQueryCommand();
+ boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
+
+ CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
+ .setQueryCommand(cmd)
+ .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
+ .setSearcher(searcher);
+
+ int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
+ docsToCollect = Math.max(docsToCollect, 1);
+
+ for (String field : groupingSpec.getFields()) {
+ SchemaField schemaField = schema.getField(field);
+ String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
+ if (topGroupsParam == null) {
+ topGroupsParam = new String[0];
+ }
+
+ List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
+ for (String topGroup : topGroupsParam) {
+ SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
+ if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
+ BytesRefBuilder builder = new BytesRefBuilder();
+ schemaField.getType().readableToIndexed(topGroup, builder);
+ searchGroup.groupValue = builder.get();
+ }
+ topGroups.add(searchGroup);
+ }
+
+ secondPhaseBuilder.addCommandField(
+ new TopGroupsFieldCommand.Builder()
+ .setField(schemaField)
+ .setGroupSort(groupingSpec.getGroupSort())
+ .setSortWithinGroup(groupingSpec.getSortWithinGroup())
+ .setFirstPhaseGroups(topGroups)
+ .setMaxDocPerGroup(docsToCollect)
+ .setNeedScores(needScores)
+ .setNeedMaxScore(needScores)
+ .build()
+ );
+ }
+
+ for (String query : groupingSpec.getQueries()) {
+ secondPhaseBuilder.addCommandField(new Builder()
+ .setDocsToCollect(docsToCollect)
+ .setSort(groupingSpec.getGroupSort())
+ .setQuery(query, rb.req)
+ .setDocSet(searcher)
+ .build()
+ );
+ }
+
+ CommandHandler commandHandler = secondPhaseBuilder.build();
+ commandHandler.execute();
+ TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
+ rsp.add("secondPhase", commandHandler.processResult(result, serializer));
+ rb.setResult(result);
+ }
+
+ private void doProcessGroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
+
+ GroupingSpecification groupingSpec = rb.getGroupingSpec();
+ assert null != groupingSpec : "GroupingSpecification is null";
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrParams params = req.getParams();
+
+ SolrIndexSearcher searcher = req.getSearcher();
+
+ QueryCommand cmd = rb.getQueryCommand();
+
+ int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
+ boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
+ Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
+ Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
+ int limitDefault = cmd.getLen(); // this is normally from "rows"
+ Grouping grouping =
+ new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
+ grouping.setGroupSort(groupingSpec.getGroupSort())
+ .setWithinGroupSort(groupingSpec.getSortWithinGroup())
+ .setDefaultFormat(groupingSpec.getResponseFormat())
+ .setLimitDefault(limitDefault)
+ .setDefaultTotalCount(defaultTotalCount)
+ .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
+ .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
+ .setGetGroupedDocSet(groupingSpec.isTruncateGroups());
+
+ if (groupingSpec.getFields() != null) {
+ for (String field : groupingSpec.getFields()) {
+ grouping.addFieldCommand(field, rb.req);
+ }
+ }
+
+ if (groupingSpec.getFunctions() != null) {
+ for (String groupByStr : groupingSpec.getFunctions()) {
+ grouping.addFunctionCommand(groupByStr, rb.req);
+ }
+ }
+
+ if (groupingSpec.getQueries() != null) {
+ for (String groupByStr : groupingSpec.getQueries()) {
+ grouping.addQueryCommand(groupByStr, rb.req);
+ }
+ }
+
+ if( rb.isNeedDocList() || rb.isDebug() ){
+ // we need a single list of the returned docs
+ cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
+ }
+
+ grouping.execute();
+ if (grouping.isSignalCacheWarning()) {
+ rsp.add(
+ "cacheWarning",
+ String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
+ );
+ }
+ rb.setResult(result);
+
+ if (grouping.mainResult != null) {
+ ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
+ rsp.addResponse(ctx);
+ rsp.getToLog().add("hits", grouping.mainResult.matches());
+ } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
+ rsp.add("grouped", result.groupedResults);
+ rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
+ }
+ }
+
+ private void doProcessUngroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException {
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrIndexSearcher searcher = req.getSearcher();
+
+ QueryCommand cmd = rb.getQueryCommand();
+
+ searcher.search(result, cmd);
+ rb.setResult(result);
+
+ ResultContext ctx = new BasicResultContext(rb);
+ rsp.addResponse(ctx);
+ rsp.getToLog().add("hits", rb.getResults().docList.matches());
+
+ if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
+ if (null != rb.getNextCursorMark()) {
+ rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
+ rb.getNextCursorMark().getSerializedTotem());
+ }
+ }
+
+ if(rb.mergeFieldHandler != null) {
+ rb.mergeFieldHandler.handleMergeFields(rb, searcher);
+ } else {
+ doFieldSortValues(rb, searcher);
+ }
+
+ doPrefetch(rb);
+ }
+
/**
* Fake scorer for a single document
*