You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@solr.apache.org by GitBox <gi...@apache.org> on 2022/02/12 21:00:52 UTC
[GitHub] [solr] mkhludnev commented on a change in pull request #623: WIP {!join cacheEventually=true ...}...
mkhludnev commented on a change in pull request #623:
URL: https://github.com/apache/solr/pull/623#discussion_r804111164
##########
File path: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
##########
@@ -194,6 +195,47 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para
@Override
public Query parse() throws SyntaxError {
+ final Query query = parseImpl();
+ // make cross core joins time-agnostic
+ // it should be ruled by param probably
+ boolean crossCoreCache = false;
+ if(localParams.getBool("cacheEventually", false)) {
Review comment:
introducing `{!join cacheEventually=true ...}...`
##########
File path: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
##########
@@ -194,6 +195,47 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para
@Override
public Query parse() throws SyntaxError {
+ final Query query = parseImpl();
+ // make cross core joins time-agnostic
+ // it should be ruled by param probably
+ boolean crossCoreCache = false;
+ if(localParams.getBool("cacheEventually", false)) {
+ if (query instanceof JoinQuery) {
+ if (((JoinQuery) query).fromCoreOpenTime != 0L) {
+ ((JoinQuery) query).fromCoreOpenTime = Long.MIN_VALUE;
+ crossCoreCache = true;
+ }
+ } else {
+ if (query instanceof ScoreJoinQParserPlugin.OtherCoreJoinQuery){
+ if (((ScoreJoinQParserPlugin.OtherCoreJoinQuery) query).fromCoreOpenTime!=0) {
+ ((ScoreJoinQParserPlugin.OtherCoreJoinQuery) query).fromCoreOpenTime = Long.MIN_VALUE;
+ crossCoreCache = true;
+ }
+ }
+ }
+ }
+ if (crossCoreCache) {
+ String fromIndex = localParams.get("fromIndex");// TODO in might be a single sharded collection
+
+ DocSet docset;
+ try {
+ WrappedQuery wrap = new WrappedQuery(query);
+ wrap.setCache(false); //bypassing searcher cache
+ @SuppressWarnings("unchecked")
+ final SolrCache<Query,DocSet> rightSideCache = (SolrCache<Query,DocSet>) req.getSearcher().getCache(fromIndex);
Review comment:
It uses left side core user cache by right side (`fromCore`) name. I suppose it's a clear convention.
##########
File path: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
##########
@@ -194,6 +196,49 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para
@Override
public Query parse() throws SyntaxError {
+ final Query query = parseImpl();
+ // make cross core joins time-agnostic
+ // it should be ruled by param probably
+ boolean crossCoreCache = false;
+ if(localParams.getBool("cacheEventually", false)) {
+ if (query instanceof JoinQuery) {
+ if (((JoinQuery) query).fromCoreOpenTime != 0L) {
+ ((JoinQuery) query).fromCoreOpenTime = Long.MIN_VALUE;
+ crossCoreCache = true;
+ }
+ } else {
+ if (query instanceof ScoreJoinQParserPlugin.OtherCoreJoinQuery){
+ if (((ScoreJoinQParserPlugin.OtherCoreJoinQuery) query).fromCoreOpenTime!=0) {
+ ((ScoreJoinQParserPlugin.OtherCoreJoinQuery) query).fromCoreOpenTime = Long.MIN_VALUE;
+ crossCoreCache = true;
+ }
+ }
+ }
+ }
+ if (crossCoreCache) {
+ String fromIndex = localParams.get("fromIndex");// TODO in might be a single sharded collection
+
+ DocSet docset;
+ try {
+ WrappedQuery wrap = new WrappedQuery(query);
+ wrap.setCache(false); //bypassing searcher cache
+ @SuppressWarnings("unchecked")
+ final SolrCache<Query,DocSet> rightSideCache = (SolrCache<Query,DocSet>) req.getSearcher().getCache(fromIndex);
+ docset = rightSideCache.computeIfAbsent(wrap, k -> req.getSearcher().getDocSet(k));
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Exception on caching " + query, e);
+ }
+ final Filter topFilter = docset.getTopFilter();
+
+ WrappedQuery wrappedCache = new WrappedQuery(topFilter);
+ wrappedCache.setCache(false);
Review comment:
return non-cached query backed on doc set cached in user cache
##########
File path: solr/core/src/test/org/apache/solr/search/RefreshCrossCoreJoinCacheFactory.java
##########
@@ -0,0 +1,66 @@
+package org.apache.solr.search;
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.RTimerTree;
+import org.apache.solr.util.RefCounted;
+
+import java.io.IOException;
+import java.security.Principal;
+import java.util.List;
+import java.util.Map;
+
+public class RefreshCrossCoreJoinCacheFactory extends UpdateRequestProcessorFactory {
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return new UpdateRequestProcessor(next) {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void processCommit(CommitUpdateCommand cmd) throws IOException {
+ super.processCommit(cmd);
+ // refresh strictly after RunUpdateProcessor
+
+ final CoreContainer coreContainer = req.getCore().getCoreContainer();
+ final List<String> loadedCoreNames = coreContainer.getLoadedCoreNames();
+ String rightSideCore = req.getCore().getName();
+ for (String leftCoreName: loadedCoreNames){
+ if (!leftCoreName.equals(rightSideCore)) {
+ final SolrCore core = coreContainer.getCore(leftCoreName);
Review comment:
Enumerates all possible "to" cores, find user cache with name "fromCore".
##########
File path: solr/core/src/test-files/solr/configsets/minimal-bckg-join/conf/solrconfig.xml
##########
@@ -0,0 +1,49 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ </updateHandler>
+
+ <updateProcessor name="refresh-join-caches" class="solr.RefreshCrossCoreJoinCacheFactory"/>
Review comment:
This guy from the right side (fromCore) refreshes left side (to) caches.
##########
File path: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
##########
@@ -194,6 +195,47 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para
@Override
public Query parse() throws SyntaxError {
+ final Query query = parseImpl();
+ // make cross core joins time-agnostic
+ // it should be ruled by param probably
+ boolean crossCoreCache = false;
+ if(localParams.getBool("cacheEventually", false)) {
+ if (query instanceof JoinQuery) {
Review comment:
two avoid those ugly `instanceof`s I'd like to unify existing JoinQP routines to unify and extract common ancestor.
##########
File path: solr/core/src/java/org/apache/solr/search/join/CrossCoreJoinCacheRegenerator.java
##########
@@ -0,0 +1,24 @@
+package org.apache.solr.search.join;
+
+import org.apache.lucene.search.Query;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.search.*;
+
+import java.io.IOException;
+import java.util.List;
+
+public class CrossCoreJoinCacheRegenerator implements CacheRegenerator {
Review comment:
goes through user cache updates doc set against new right side searcher
##########
File path: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
##########
@@ -194,6 +195,47 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para
@Override
public Query parse() throws SyntaxError {
+ final Query query = parseImpl();
+ // make cross core joins time-agnostic
+ // it should be ruled by param probably
+ boolean crossCoreCache = false;
+ if(localParams.getBool("cacheEventually", false)) {
+ if (query instanceof JoinQuery) {
+ if (((JoinQuery) query).fromCoreOpenTime != 0L) {
+ ((JoinQuery) query).fromCoreOpenTime = Long.MIN_VALUE;
Review comment:
I need these queries match to each other for cache hit regardless of commit in `fromCore`. It can be done by setting some additional flag and changing `equals`&`hashCode`. WDYT?
##########
File path: solr/core/src/test-files/solr/collection1/conf/solrconfig-crosscore-join-cache.xml
##########
@@ -0,0 +1,534 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- This is a "kitchen sink" config file that tests can use.
+ When writting a new test, feel free to add *new* items (plugins,
+ config options, etc...) as long as they don't break any existing
+ tests. if you need to test something esoteric please add a new
+ "solrconfig-your-esoteric-purpose.xml" config file.
+
+ Note in particular that this test is used by MinimalSchemaTest so
+ Anything added to this file needs to work correctly even if there
+ is now uniqueKey or defaultSearch Field.
+ -->
+
+<config>
+
+ <jmx />
+
+ <!-- Used to specify an alternate directory to hold all index data.
+ It defaults to "index" if not present, and should probably
+ not be changed if replication is in use. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <!-- The DirectoryFactory to use for indexes.
+ solr.StandardDirectoryFactory, the default, is filesystem based.
+ solr.RAMDirectoryFactory is memory based and not persistent. -->
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
+ <double name="maxWriteMBPerSecDefault">1000000</double>
+ <double name="maxWriteMBPerSecFlush">2000000</double>
+ <double name="maxWriteMBPerSecMerge">3000000</double>
+ <double name="maxWriteMBPerSecRead">4000000</double>
+ <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
+ <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+ <bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
+ <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
+ <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
+ <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
+ </directoryFactory>
+
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <statsCache class="${solr.statsCache:}"/>
+
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <autoCommit>
+ <maxTime>${solr.autoCommit.maxTime:-1}</maxTime>
+ </autoCommit>
+
+ <!-- autocommit pending docs if certain criteria are met
+ <autoCommit>
+ <maxDocs>10000</maxDocs>
+ <maxTime>3600000</maxTime>
+ </autoCommit>
+ -->
+
+ <updateLog enable="${enable.update.log:true}">
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+
+ </updateHandler>
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or wildcard queries that expand to big boolean
+ queries. An exception is thrown if exceeded.
+ -->
+ <maxBooleanClauses>${solr.max.booleanClauses:1024}</maxBooleanClauses>
+
+ <!-- Cache specification for Filters or DocSets - unordered set of *all* documents
+ that match a particular query.
+ -->
+ <filterCache
+ size="512"
+ initialSize="512"
+ autowarmCount="2"
+ async="${solr.filterCache.async:false}"/>
+
+ <queryResultCache
+ size="512"
+ initialSize="512"
+ autowarmCount="2"/>
+
+ <documentCache
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <cache name="perSegFilter"
+ class="solr.CaffeineCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10" />
+
+ <cache name="fromCore"
+ class="solr.CaffeineCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10"
Review comment:
warm all entries
##########
File path: solr/core/src/test/org/apache/solr/search/RefreshCrossCoreJoinCacheFactory.java
##########
@@ -0,0 +1,66 @@
+package org.apache.solr.search;
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.RTimerTree;
+import org.apache.solr.util.RefCounted;
+
+import java.io.IOException;
+import java.security.Principal;
+import java.util.List;
+import java.util.Map;
+
+public class RefreshCrossCoreJoinCacheFactory extends UpdateRequestProcessorFactory {
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return new UpdateRequestProcessor(next) {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void processCommit(CommitUpdateCommand cmd) throws IOException {
+ super.processCommit(cmd);
+ // refresh strictly after RunUpdateProcessor
+
+ final CoreContainer coreContainer = req.getCore().getCoreContainer();
+ final List<String> loadedCoreNames = coreContainer.getLoadedCoreNames();
+ String rightSideCore = req.getCore().getName();
+ for (String leftCoreName: loadedCoreNames){
+ if (!leftCoreName.equals(rightSideCore)) {
+ final SolrCore core = coreContainer.getCore(leftCoreName);
+ final RefCounted<SolrIndexSearcher> leftSearcher = core.getSearcher();
+ try {
+ @SuppressWarnings("rawtypes")
+ final SolrCache joinCache = leftSearcher.get().getCache(rightSideCore);
+ if (joinCache != null) {
+ // this is necessary for classic join query, which checks SRI, i don't know why.
+ SolrQueryRequest leftReq = new LocalSolrQueryRequest(core,req.getParams()) {
+ @Override public SolrIndexSearcher getSearcher() { return leftSearcher.get(); }
+ @Override public void close() { }
+ };
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(leftReq, rsp));
+ try {
+ joinCache.warm(leftSearcher.get(), joinCache);
Review comment:
Self warm left side caches picking fresh "fromCore" searcher.
##########
File path: solr/core/src/test-files/solr/collection1/conf/solrconfig-crosscore-join-cache.xml
##########
@@ -0,0 +1,534 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- This is a "kitchen sink" config file that tests can use.
+ When writting a new test, feel free to add *new* items (plugins,
+ config options, etc...) as long as they don't break any existing
+ tests. if you need to test something esoteric please add a new
+ "solrconfig-your-esoteric-purpose.xml" config file.
+
+ Note in particular that this test is used by MinimalSchemaTest so
+ Anything added to this file needs to work correctly even if there
+ is now uniqueKey or defaultSearch Field.
+ -->
+
+<config>
+
+ <jmx />
+
+ <!-- Used to specify an alternate directory to hold all index data.
+ It defaults to "index" if not present, and should probably
+ not be changed if replication is in use. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <!-- The DirectoryFactory to use for indexes.
+ solr.StandardDirectoryFactory, the default, is filesystem based.
+ solr.RAMDirectoryFactory is memory based and not persistent. -->
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
+ <double name="maxWriteMBPerSecDefault">1000000</double>
+ <double name="maxWriteMBPerSecFlush">2000000</double>
+ <double name="maxWriteMBPerSecMerge">3000000</double>
+ <double name="maxWriteMBPerSecRead">4000000</double>
+ <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
+ <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+ <bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
+ <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
+ <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
+ <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
+ </directoryFactory>
+
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <statsCache class="${solr.statsCache:}"/>
+
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <autoCommit>
+ <maxTime>${solr.autoCommit.maxTime:-1}</maxTime>
+ </autoCommit>
+
+ <!-- autocommit pending docs if certain criteria are met
+ <autoCommit>
+ <maxDocs>10000</maxDocs>
+ <maxTime>3600000</maxTime>
+ </autoCommit>
+ -->
+
+ <updateLog enable="${enable.update.log:true}">
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+
+ </updateHandler>
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or wildcard queries that expand to big boolean
+ queries. An exception is thrown if exceeded.
+ -->
+ <maxBooleanClauses>${solr.max.booleanClauses:1024}</maxBooleanClauses>
+
+ <!-- Cache specification for Filters or DocSets - unordered set of *all* documents
+ that match a particular query.
+ -->
+ <filterCache
+ size="512"
+ initialSize="512"
+ autowarmCount="2"
+ async="${solr.filterCache.async:false}"/>
+
+ <queryResultCache
+ size="512"
+ initialSize="512"
+ autowarmCount="2"/>
+
+ <documentCache
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <cache name="perSegFilter"
+ class="solr.CaffeineCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10" />
+
+ <cache name="fromCore"
Review comment:
Here's what we need. User Cache named as right side core.
##########
File path: solr/core/src/test/org/apache/solr/search/RefreshCrossCoreJoinCacheFactory.java
##########
@@ -0,0 +1,66 @@
+package org.apache.solr.search;
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.RTimerTree;
+import org.apache.solr.util.RefCounted;
+
+import java.io.IOException;
+import java.security.Principal;
+import java.util.List;
+import java.util.Map;
+
+public class RefreshCrossCoreJoinCacheFactory extends UpdateRequestProcessorFactory {
Review comment:
kicked on fromCore
##########
File path: solr/core/src/test-files/solr/collection1/conf/solrconfig-crosscore-join-cache.xml
##########
@@ -0,0 +1,534 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- This is a "kitchen sink" config file that tests can use.
+ When writting a new test, feel free to add *new* items (plugins,
+ config options, etc...) as long as they don't break any existing
+ tests. if you need to test something esoteric please add a new
+ "solrconfig-your-esoteric-purpose.xml" config file.
+
+ Note in particular that this test is used by MinimalSchemaTest so
+ Anything added to this file needs to work correctly even if there
+ is now uniqueKey or defaultSearch Field.
+ -->
+
+<config>
+
+ <jmx />
+
+ <!-- Used to specify an alternate directory to hold all index data.
+ It defaults to "index" if not present, and should probably
+ not be changed if replication is in use. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <!-- The DirectoryFactory to use for indexes.
+ solr.StandardDirectoryFactory, the default, is filesystem based.
+ solr.RAMDirectoryFactory is memory based and not persistent. -->
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
+ <double name="maxWriteMBPerSecDefault">1000000</double>
+ <double name="maxWriteMBPerSecFlush">2000000</double>
+ <double name="maxWriteMBPerSecMerge">3000000</double>
+ <double name="maxWriteMBPerSecRead">4000000</double>
+ <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
+ <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+ <bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
+ <bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
+ <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
+ <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
+ </directoryFactory>
+
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <statsCache class="${solr.statsCache:}"/>
+
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <autoCommit>
+ <maxTime>${solr.autoCommit.maxTime:-1}</maxTime>
+ </autoCommit>
+
+ <!-- autocommit pending docs if certain criteria are met
+ <autoCommit>
+ <maxDocs>10000</maxDocs>
+ <maxTime>3600000</maxTime>
+ </autoCommit>
+ -->
+
+ <updateLog enable="${enable.update.log:true}">
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+
+ </updateHandler>
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or wildcard queries that expand to big boolean
+ queries. An exception is thrown if exceeded.
+ -->
+ <maxBooleanClauses>${solr.max.booleanClauses:1024}</maxBooleanClauses>
+
+ <!-- Cache specification for Filters or DocSets - unordered set of *all* documents
+ that match a particular query.
+ -->
+ <filterCache
+ size="512"
+ initialSize="512"
+ autowarmCount="2"
+ async="${solr.filterCache.async:false}"/>
+
+ <queryResultCache
+ size="512"
+ initialSize="512"
+ autowarmCount="2"/>
+
+ <documentCache
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <cache name="perSegFilter"
+ class="solr.CaffeineCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10" />
+
+ <cache name="fromCore"
+ class="solr.CaffeineCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10"
+ regenerator="org.apache.solr.search.join.CrossCoreJoinCacheRegenerator"/>
Review comment:
it can self warm to refresh docsets
##########
File path: solr/core/src/test/org/apache/solr/search/TestBackgroundJoinWarmer.java
##########
@@ -0,0 +1,183 @@
+package org.apache.solr.search;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.servlet.DirectSolrConnection;
+import org.apache.solr.util.RefCounted;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class TestBackgroundJoinWarmer extends SolrTestCaseJ4 {
+
+ private static SolrCore fromCore;
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
+ System.setProperty("solr.filterCache.async", "true");
+
+ initCore("solrconfig-crosscore-join-cache.xml", "schema12.xml", TEST_HOME(), "collection1");
+ final CoreContainer coreContainer = h.getCoreContainer();
+
+ fromCore = coreContainer.create("fromCore", ImmutableMap.of("configSet", "minimal-bckg-join"));
+
+ assertU(add(doc("id", "1", "id_s_dv", "1", "name", "john", "title", "Director", "dept_s", "Engineering")));
+ assertU(add(doc("id", "2", "id_s_dv", "2", "name", "mark", "title", "VP", "dept_s", "Marketing")));
+ assertU(add(doc("id", "3", "id_s_dv", "3", "name", "nancy", "title", "MTS", "dept_s", "Sales")));
+ assertU(add(doc("id", "4", "id_s_dv", "4", "name", "dave", "title", "MTS", "dept_s", "Support", "dept_s", "Engineering")));
+ assertU(add(doc("id", "5", "id_s_dv", "5", "name", "tina", "title", "VP", "dept_s", "Engineering")));
+ assertU(commit());
+
+ update(fromCore, add(doc("id", "10", "id_s_dv", "10", "dept_id_s", "Engineering", "text", "These guys develop stuff", "cat", "dev")));
+ update(fromCore, add(doc("id", "11", "id_s_dv", "11", "dept_id_s", "Marketing", "text", "These guys make you look good")));
+ update(fromCore, add(doc("id", "12", "id_s_dv", "12", "dept_id_s", "Sales", "text", "These guys sell stuff")));
+ update(fromCore, add(doc("id", "13", "id_s_dv", "13", "dept_id_s", "Support", "text", "These guys help customers")));
+ update(fromCore, commit());
+
+ }
+
+
+ public static String update(SolrCore core, String xml) throws Exception {
+ return update(core, xml, null);
+ }
+
+ public static String update(SolrCore core, String xml, SolrParams params) throws Exception {
+ DirectSolrConnection connection = new DirectSolrConnection(core);
+ SolrRequestHandler handler = core.getRequestHandler("/update");
+ return connection.request(handler, params, xml);
+ }
+
+ @Test
+ public void testScoreJoin() throws Exception {
+ doTestJoin("{!join "+(random().nextBoolean() ? "score=none" : ""));
+ }
+
+ void doTestJoin(String joinPrefix) throws Exception {
+ long toSearcherOpenTime = getSearcherOpenTime();
+ assertUserCacheStats("fromCore", "size", 0);
+ assertTrue(toSearcherOpenTime>0L);
+ for(long hits=0L; hits<2L; hits++) {
+ assertJQ(req(qOrFq(joinPrefix + " from=dept_id_s to=dept_s cacheEventually=true fromIndex=fromCore}cat:dev", "fl", "id",
+ "debugQuery", random().nextBoolean() ? "true" : "false"))
+ , "/response=={'numFound':3,'start':0,'numFoundExact':true,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
+ );
+ // assert user cache hit
+ assertUserCacheStats("fromCore", "size", 1,
+ "hits", hits, "cumulative_hits", hits, //0, 1
+ "inserts", 1L , "cumulative_inserts", 1L);
+ }
+ update(fromCore, delQ("id:10"));
+ update(fromCore, add(doc("id", "10", "id_s_dv", "10", "dept_id_s", "Marketing",
+ "text", "now they sell", "cat", "dev")));
+ update(fromCore, commit());
+ // change in fromCore isn't visible due to user cache cache
+ assertJQ(req(qOrFq( joinPrefix + " from=dept_id_s to=dept_s cacheEventually=true fromIndex=fromCore}cat:dev", "fl", "id",
+ "debugQuery", random().nextBoolean() ? "true" : "false"))
+ , "/response=={'numFound':3,'start':0,'numFoundExact':true,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
+ );
+ // nothing have changed ...
+ assertUserCacheStats("fromCore", "size", 1,
+ "hits", 2L, "cumulative_hits", 2L, // inc from the last check
+ "inserts", 1L , "cumulative_inserts", 1L);
+
+ if (true) {
+ update(fromCore, commit(), new MapSolrParams(Map.of("post-processor", "refresh-join-caches")));
+ assertUserCacheStats("fromCore", "size", 1,
+ "hits", 0L,
+ "cumulative_hits", 0L, // that's because it warms itself
+ "inserts", 0L ,
+ "cumulative_inserts", 0L); // that's because it warms itself
+ } else { // commit into main index refreshes cache as well
+ assertU(add(doc("id", "99999999")));
+ assertU(commit()); // TODO this causes an error.
+ // cached join leaks as query result key,
Review comment:
This is a problem so far. Doc set from old searcher breaks warming of fresh searcher. Have no solution.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@solr.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@solr.apache.org
For additional commands, e-mail: issues-help@solr.apache.org