You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ge...@apache.org on 2023/10/19 18:53:03 UTC
[solr-sandbox] branch main updated: Ignore producer URP (if disabled) in standalone mode (#79)
This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/main by this push:
new e978a7a Ignore producer URP (if disabled) in standalone mode (#79)
e978a7a is described below
commit e978a7a4c1349cfcd6511663a2b895b0e972c921
Author: Jason Gerlowski <ge...@apache.org>
AuthorDate: Thu Oct 19 14:52:59 2023 -0400
Ignore producer URP (if disabled) in standalone mode (#79)
XDC really only makes sense in a SolrCloud context. But all the same,
its nice to handle MirroringUpdateRequestProcessorFactory more
gracefully if someone does accidentally configure their standalone core
to use the XDC producer URP.
This commit tweaks MirroringUpdateRequestProcessorFactory's
initialization so that in standalone mode it emits a warning about the
misconfiguration if disabled, and fails core init/load entirely if the
Mirroring URP is 'enabled=true'.
---
.../MirroringUpdateRequestProcessorFactory.java | 25 +++--
.../crossdc/CrossDCProducerSolrStandaloneTest.java | 101 +++++++++++++++++
.../conf/solrconfig-producerdisabled.xml | 119 +++++++++++++++++++++
3 files changed, 239 insertions(+), 6 deletions(-)
diff --git a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
index 0077339..80ce2a1 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
@@ -96,6 +96,7 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
}
}
+
private static class MyCloseHook extends CloseHook {
private final Closer closer;
@@ -129,10 +130,7 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
}
- @Override
- public void inform(SolrCore core) {
- log.info("KafkaRequestMirroringHandler inform enabled={}", this.enabled);
-
+ private void lookupPropertyOverridesInZk(SolrCore core) {
log.info("Producer startup config properties before adding additional properties from Zookeeper={}", properties);
try {
@@ -140,7 +138,7 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
ConfUtil.fillProperties(solrZkClient, properties);
applyArgsOverrides();
CollectionProperties cp = new CollectionProperties(solrZkClient);
- Map<String,String> collectionProperties = cp.getCollectionProperties(core.getCoreDescriptor().getCollectionName());
+ Map<String,String> collectionProperties = cp.getCollectionProperties(core.getCoreDescriptor().getCollectionName());
for (ConfigProperty configKey : KafkaCrossDcConf.CONFIG_PROPERTIES) {
String val = collectionProperties.get("crossdc." + configKey.getKey());
if (val != null && !val.isBlank()) {
@@ -159,6 +157,21 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
log.error("Exception looking for CrossDC configuration in Zookeeper", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Exception looking for CrossDC configuration in Zookeeper", e);
}
+ }
+
+ @Override
+ public void inform(SolrCore core) {
+ log.info("KafkaRequestMirroringHandler inform enabled={}", this.enabled);
+
+ if (core.getCoreContainer().isZooKeeperAware()) {
+ lookupPropertyOverridesInZk(core);
+ } else {
+ applyArgsOverrides();
+ if (enabled) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, getClass().getSimpleName() + " only supported in SolrCloud mode; please disable or remove from solrconfig.xml");
+ }
+ log.warn("Core '{}' was configured to use a disabled {}, but {} is only supported in SolrCloud deployments. A NoOp processor will be used instead", core.getName(), this.getClass().getSimpleName(), this.getClass().getSimpleName());
+ }
if (!enabled) {
return;
@@ -240,7 +253,7 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)), doMirroring ? mirroringHandler : null);
}
- private static class NoOpUpdateRequestProcessor extends UpdateRequestProcessor {
+ public static class NoOpUpdateRequestProcessor extends UpdateRequestProcessor {
NoOpUpdateRequestProcessor(UpdateRequestProcessor next) {
super(next);
}
diff --git a/crossdc-producer/src/test/java/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java b/crossdc-producer/src/test/java/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java
new file mode 100644
index 0000000..8a0c3fd
--- /dev/null
+++ b/crossdc-producer/src/test/java/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.crossdc;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.core.SolrCoreInitializationException;
+import org.apache.solr.update.processor.MirroringUpdateRequestProcessorFactory;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Unit test validating that {@link MirroringUpdateRequestProcessorFactory} responds appropriately in Solr is running in standalone mode.
+ */
+public class CrossDCProducerSolrStandaloneTest extends SolrTestCaseJ4 {
+
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ h.close();
+ }
+
+ @Test
+ public void testSolrStandaloneQuietlyNoopsDisabledProducer() throws Exception {
+ try (final EmbeddedSolrServer server = createProducerCoreWithProperties("solrconfig-producerdisabled.xml", "producerDisabledCore")) {
+ try (final var producerDisabledCore = server.getCoreContainer().getCore("producerDisabledCore")) {
+ assertNotNull(producerDisabledCore);
+ final var mirrorChain = producerDisabledCore.getUpdateProcessingChain("mirrorUpdateChain");
+ assertNotNull(mirrorChain);
+ final var updateProcessorList = mirrorChain.getProcessors();
+ final var mirroringFactoryOption = updateProcessorList.stream()
+ .filter(pf -> pf instanceof MirroringUpdateRequestProcessorFactory)
+ .findFirst();
+ assertTrue("No mirroring factory found in " + updateProcessorList, mirroringFactoryOption.isPresent());
+ final var mirroringFactory = mirroringFactoryOption.get();
+
+ final var mirroringProcessorInstance = mirroringFactory.getInstance(null, null, null);
+ assertEquals(MirroringUpdateRequestProcessorFactory.NoOpUpdateRequestProcessor.class, mirroringProcessorInstance.getClass());
+ }
+ }
+ }
+
+ @Test
+ public void testEnabledProcessorFailsCoreInitInSolrStandalone() throws Exception {
+ try (final EmbeddedSolrServer server = createProducerCoreWithProperties("solrconfig.xml", "producerEnabledCore")) {
+ expectThrows(SolrCoreInitializationException.class, () -> {
+ final var core = server.getCoreContainer().getCore("producerEnabledCore");
+ // Should be preempted by exception in line above, but ensures the core is closed in case the test is about to fail
+ core.close();
+ });
+ }
+ }
+
+ private static EmbeddedSolrServer createProducerCoreWithProperties(String solrConfigName, String coreName) throws Exception {
+ Path tmpHome = createTempDir("tmp-home");
+ Path coreDir = tmpHome.resolve(coreName);
+ populateCoreDirectory("src/test/resources/configs/cloud-minimal/conf", solrConfigName, coreDir.toFile());
+ initCore(
+ "solrconfig.xml", "schema.xml", tmpHome.toAbsolutePath().toString(), coreName);
+
+ return new EmbeddedSolrServer(h.getCoreContainer(), coreName);
+ }
+
+ /**
+ * Copy configset files to a specified location
+ *
+ * @param sourceLocation the location of schema and solrconfig files to copy
+ * @param solrConfigName the name of the solrconfig file to use for this core
+ * @param coreDirectory an empty preexisting location use as a core directory.
+ * @throws IOException
+ */
+ private static void populateCoreDirectory(String sourceLocation, String solrConfigName, File coreDirectory) throws IOException {
+ File subHome = new File(coreDirectory, "conf");
+ if (! coreDirectory.exists()) {
+ assertTrue("Failed to make subdirectory ", coreDirectory.mkdirs());
+ }
+ Files.createFile(coreDirectory.toPath().resolve("core.properties"));
+ FileUtils.copyFile(new File(sourceLocation, "schema.xml"), new File(subHome, "schema.xml"));
+ FileUtils.copyFile(new File(sourceLocation, solrConfigName), new File(subHome, "solrconfig.xml"));
+ }
+}
diff --git a/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig-producerdisabled.xml b/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig-producerdisabled.xml
new file mode 100644
index 0000000..8de9263
--- /dev/null
+++ b/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig-producerdisabled.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <indexConfig>
+ <mergePolicyFactory class="${mergePolicyFactory:org.apache.solr.index.TieredMergePolicyFactory}">
+ <int name="maxMergeAtOnce">${maxMergeAtOnce:10}</int>
+ <int name="segmentsPerTier">${segmentsPerTier:10}</int>
+ <double name="noCFSRatio">${noCFSRatio:.1}</double>
+ </mergePolicyFactory>
+
+ <useCompoundFile>${useCompoundFile:true}</useCompoundFile>
+
+ <ramBufferSizeMB>${ramBufferSizeMB:160}</ramBufferSizeMB>
+ <maxBufferedDocs>${maxBufferedDocs:250000}</maxBufferedDocs> <!-- Force the common case to flush by doc count -->
+ <!-- <ramPerThreadHardLimitMB>60</ramPerThreadHardLimitMB> -->
+
+ <!-- <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler">
+ <int name="maxThreadCount">6</int>
+ <int name="maxMergeCount">8</int>
+ <bool name="ioThrottle">false</bool>
+ </mergeScheduler> -->
+
+ <writeLockTimeout>1000</writeLockTimeout>
+ <commitLockTimeout>10000</commitLockTimeout>
+
+ <!-- this sys property is not set by SolrTestCaseJ4 because almost all tests should
+ use the single process lockType for speed - but tests that explicitly need
+ to vary the lockType canset it as needed.
+ -->
+ <lockType>${lockType:single}</lockType>
+
+ <infoStream>${infostream:false}</infoStream>
+
+ </indexConfig>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ <autoCommit>
+ <maxTime>${autoCommit.maxTime:60000}</maxTime>
+ </autoCommit>
+ <updateLog class="${ulog:solr.UpdateLog}" enable="${enable.update.log:true}"/>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+
+ <query>
+ <queryResultCache
+ enabled="${queryResultCache.enabled:false}"
+ class="${queryResultCache.class:solr.CaffeineCache}"
+ size="${queryResultCache.size:0}"
+ initialSize="${queryResultCache.initialSize:0}"
+ autowarmCount="${queryResultCache.autowarmCount:0}"/>
+ <documentCache
+ enabled="${documentCache.enabled:false}"
+ class="${documentCache.class:solr.CaffeineCache}"
+ size="${documentCache.size:0}"
+ initialSize="${documentCache.initialSize:0}"
+ autowarmCount="${documentCache.autowarmCount:0}"/>
+ <filterCache
+ enabled ="${filterCache.enabled:false}"
+ class="${filterCache.class:solr.CaffeineCache}"
+ size="${filterCache.size:1}"
+ initialSize="${filterCache.initialSize:1}"
+ autowarmCount="${filterCache.autowarmCount:0}"
+ async="${filterCache.async:false}"/>
+ <cache name="myPerSegmentCache"
+ enabled="${myPerSegmentCache.enabled:false}"
+ class="${myPerSegmentCache.class:solr.CaffeineCache}"
+ size="${myPerSegmentCache.size:0}"
+ initialSize="${myPerSegmentCache.initialSize:0}"
+ autowarmCount="${myPerSegmentCache.autowarmCount:0}"/>
+ </query>
+
+ <updateRequestProcessorChain name="mirrorUpdateChain" default="true">
+ <processor class="org.apache.solr.update.processor.MirroringUpdateRequestProcessorFactory">
+ <bool name="enabled">false</bool>
+ </processor>
+ <processor class="solr.LogUpdateProcessorFactory" />
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+
+</config>