You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ma...@apache.org on 2022/04/22 06:39:19 UTC
[solr-sandbox] branch crossdc-wip updated: Support for integration tests with SolrCloud. (#6)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch crossdc-wip
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/crossdc-wip by this push:
new 0be4c64 Support for integration tests with SolrCloud. (#6)
0be4c64 is described below
commit 0be4c64d89527b24e24bd922275c1c66f0038481
Author: Mark Robert Miller <ma...@apache.org>
AuthorDate: Fri Apr 22 01:39:14 2022 -0500
Support for integration tests with SolrCloud. (#6)
I've had a few little issues that I've been chasing, so to get something in, I pulled out kafka and some other items and setup a minimal set of changes to use SolrCloud in tests and I have a simple test that runs the same messsage processor test but with a real cloud instance instead of a mock. Hang on a moment and I'll get kafka and the rest back in.
---
.gitignore | 1 +
crossdc-consumer/build.gradle | 5 +
.../configs/cloud-minimal/conf/schema.xml | 54 ++++++++++
.../configs/cloud-minimal/conf/solrconfig.xml | 112 +++++++++++++++++++++
crossdc-consumer/src/resources/log4j2.xml | 42 ++++++++
.../org/apache/solr/crossdc/IntegrationTest.java | 91 +++++++++++++++++
.../apache/solr/crossdc/TestMessageProcessor.java | 2 +-
7 files changed, 306 insertions(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 7013eda..4daa5b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@
# Ignore Gradle build output directory
build
+out
diff --git a/crossdc-consumer/build.gradle b/crossdc-consumer/build.gradle
index 58f3dac..9366996 100644
--- a/crossdc-consumer/build.gradle
+++ b/crossdc-consumer/build.gradle
@@ -29,6 +29,7 @@ repositories {
dependencies {
implementation group: 'org.apache.solr', name: 'solr-solrj', version: '8.11.1'
+ implementation 'org.slf4j:slf4j-api'
api 'org.eclipse.jetty:jetty-http:9.4.41.v20210516'
api 'org.eclipse.jetty:jetty-server:9.4.41.v20210516'
api 'org.eclipse.jetty:jetty-servlet:9.4.41.v20210516'
@@ -40,6 +41,10 @@ dependencies {
testImplementation('org.mockito:mockito-core:4.3.1', {
exclude group: "net.bytebuddy", module: "byte-buddy-agent"
})
+ testImplementation group: 'org.apache.solr', name: 'solr-core', version: '8.11.1'
+ testImplementation group: 'org.apache.solr', name: 'solr-test-framework', version: '8.11.1'
+ testImplementation 'org.apache.kafka:kafka_2.13:2.8.1'
+
}
subprojects {
group "org.apache.solr"
diff --git a/crossdc-consumer/src/resources/configs/cloud-minimal/conf/schema.xml b/crossdc-consumer/src/resources/configs/cloud-minimal/conf/schema.xml
new file mode 100644
index 0000000..bc4676c
--- /dev/null
+++ b/crossdc-consumer/src/resources/configs/cloud-minimal/conf/schema.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="string" class="solr.StrField" docValues="true"/>
+ <fieldType name="int" class="org.apache.solr.schema.IntPointField" docValues="true" omitNorms="true"
+ positionIncrementGap="0"/>
+ <fieldType name="long" class="org.apache.solr.schema.LongPointField" docValues="true" omitNorms="true"
+ positionIncrementGap="0"/>
+ <fieldType name="float" class="org.apache.solr.schema.FloatPointField" docValues="true" omitNorms="true"
+ positionIncrementGap="0"/>
+ <fieldType name="double" class="org.apache.solr.schema.DoublePointField" docValues="true" omitNorms="true"
+ positionIncrementGap="0"/>
+ <fieldType name="date" class="org.apache.solr.schema.DatePointField" docValues="true" omitNorms="true"
+ positionIncrementGap="0"/>
+ <fieldType name="text" class="solr.TextField">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- for versioning -->
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="id" type="string" indexed="true" stored="true"/>
+ <field name="text" type="text" indexed="true" stored="false"/>
+
+ <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="false"/>
+ <dynamicField name="*_t" type="text" indexed="true" stored="false"/>
+ <dynamicField name="*_i" type="int" indexed="false" stored="false"/>
+ <dynamicField name="*_l" type="long" indexed="false" stored="false"/>
+ <dynamicField name="*_f" type="float" indexed="false" stored="false"/>
+ <dynamicField name="*_d" type="double" indexed="false" stored="false"/>
+ <dynamicField name="*_dt" type="date" indexed="false" stored="false"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/crossdc-consumer/src/resources/configs/cloud-minimal/conf/solrconfig.xml b/crossdc-consumer/src/resources/configs/cloud-minimal/conf/solrconfig.xml
new file mode 100644
index 0000000..20caf3b
--- /dev/null
+++ b/crossdc-consumer/src/resources/configs/cloud-minimal/conf/solrconfig.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <indexConfig>
+ <mergePolicyFactory class="${mergePolicyFactory:org.apache.solr.index.TieredMergePolicyFactory}">
+ <int name="maxMergeAtOnce">${maxMergeAtOnce:10}</int>
+ <int name="segmentsPerTier">${segmentsPerTier:10}</int>
+ <double name="noCFSRatio">${noCFSRatio:.1}</double>
+ </mergePolicyFactory>
+
+ <useCompoundFile>${useCompoundFile:true}</useCompoundFile>
+
+ <ramBufferSizeMB>${ramBufferSizeMB:160}</ramBufferSizeMB>
+ <maxBufferedDocs>${maxBufferedDocs:250000}</maxBufferedDocs> <!-- Force the common case to flush by doc count -->
+ <!-- <ramPerThreadHardLimitMB>60</ramPerThreadHardLimitMB> -->
+
+ <!-- <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler">
+ <int name="maxThreadCount">6</int>
+ <int name="maxMergeCount">8</int>
+ <bool name="ioThrottle">false</bool>
+ </mergeScheduler> -->
+
+ <writeLockTimeout>1000</writeLockTimeout>
+ <commitLockTimeout>10000</commitLockTimeout>
+
+ <!-- this sys property is not set by SolrTestCaseJ4 because almost all tests should
+ use the single process lockType for speed - but tests that explicitly need
+ to vary the lockType canset it as needed.
+ -->
+ <lockType>${lockType:single}</lockType>
+
+ <infoStream>${infostream:false}</infoStream>
+
+ </indexConfig>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ <autoCommit>
+ <maxTime>${autoCommit.maxTime:60000}</maxTime>
+ </autoCommit>
+ <updateLog class="${ulog:solr.UpdateLog}" enable="${enable.update.log:true}"/>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+
+ <query>
+ <queryResultCache
+ enabled="${queryResultCache.enabled:false}"
+ class="${queryResultCache.class:solr.CaffeineCache}"
+ size="${queryResultCache.size:0}"
+ initialSize="${queryResultCache.initialSize:0}"
+ autowarmCount="${queryResultCache.autowarmCount:0}"/>
+ <documentCache
+ enabled="${documentCache.enabled:false}"
+ class="${documentCache.class:solr.CaffeineCache}"
+ size="${documentCache.size:0}"
+ initialSize="${documentCache.initialSize:0}"
+ autowarmCount="${documentCache.autowarmCount:0}"/>
+ <filterCache
+ enabled ="${filterCache.enabled:false}"
+ class="${filterCache.class:solr.CaffeineCache}"
+ size="${filterCache.size:1}"
+ initialSize="${filterCache.initialSize:1}"
+ autowarmCount="${filterCache.autowarmCount:0}"
+ async="${filterCache.async:false}"/>
+ <cache name="myPerSegmentCache"
+ enabled="${myPerSegmentCache.enabled:false}"
+ class="${myPerSegmentCache.class:solr.CaffeineCache}"
+ size="${myPerSegmentCache.size:0}"
+ initialSize="${myPerSegmentCache.initialSize:0}"
+ autowarmCount="${myPerSegmentCache.autowarmCount:0}"/>
+ </query>
+
+</config>
+
diff --git a/crossdc-consumer/src/resources/log4j2.xml b/crossdc-consumer/src/resources/log4j2.xml
new file mode 100644
index 0000000..96f69f1
--- /dev/null
+++ b/crossdc-consumer/src/resources/log4j2.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!-- We're configuring testing to be synchronous due to "logging polution", see SOLR-13268 -->
+<Configuration>
+ <Appenders>
+ <Console name="STDERR" target="SYSTEM_ERR">
+ <PatternLayout>
+ <Pattern>
+ %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{
+ =>%ex{short}}}{10240}%n
+ </Pattern>
+ </PatternLayout>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
+ <Logger name="org.apache.zookeeper" level="WARN"/>
+ <Logger name="org.apache.hadoop" level="WARN"/>
+ <Logger name="org.apache.directory" level="WARN"/>
+ <Logger name="org.apache.solr.hadoop" level="INFO"/>
+ <Logger name="org.eclipse.jetty" level="INFO"/>
+
+ <Root level="INFO">
+ <AppenderRef ref="STDERR"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/crossdc-consumer/src/test/java/org/apache/solr/crossdc/IntegrationTest.java b/crossdc-consumer/src/test/java/org/apache/solr/crossdc/IntegrationTest.java
new file mode 100644
index 0000000..695ebab
--- /dev/null
+++ b/crossdc-consumer/src/test/java/org/apache/solr/crossdc/IntegrationTest.java
@@ -0,0 +1,91 @@
+package org.apache.solr.crossdc;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.crossdc.common.MirroredSolrRequest;
+import org.apache.solr.crossdc.messageprocessor.SolrMessageProcessor;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.spy;
+
+public class IntegrationTest extends SolrCloudTestCase {
+ static final String VERSION_FIELD = "_version_";
+
+ protected static volatile MiniSolrCloudCluster cluster1;
+ protected static volatile MiniSolrCloudCluster cluster2;
+ private static SolrMessageProcessor processor;
+
+ private static ResubmitBackoffPolicy backoffPolicy = spy(new TestMessageProcessor.NoOpResubmitBackoffPolicy());
+
+ @BeforeClass
+ public static void setupIntegrationTest() throws Exception {
+
+ cluster1 =
+ new SolrCloudTestCase.Builder(2, createTempDir())
+ .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf").toPath())
+ .configure();
+
+ processor = new SolrMessageProcessor(cluster1.getSolrClient(), backoffPolicy);
+ }
+
+ @AfterClass
+ public static void tearDownIntegrationTest() throws Exception {
+ if (cluster != null) {
+ cluster1.shutdown();
+ }
+ }
+
+ public void testDocumentSanitization() {
+ UpdateRequest request = spy(new UpdateRequest());
+
+ // Add docs with and without version
+ request.add(new SolrInputDocument() {
+ {
+ setField("id", 1);
+ setField(VERSION_FIELD, 1);
+ }
+ });
+ request.add(new SolrInputDocument() {
+ {
+ setField("id", 2);
+ }
+ });
+
+ // Delete by id with and without version
+ request.deleteById("1");
+ request.deleteById("2", 10L);
+
+ request.setParam("shouldMirror", "true");
+ // The response is irrelevant, but it will fail because mocked server returns null when processing
+ processor.handleItem(new MirroredSolrRequest(request));
+
+ // After processing, check that all version fields are stripped
+ for (SolrInputDocument doc : request.getDocuments()) {
+ assertNull("Doc still has version", doc.getField(VERSION_FIELD));
+ }
+
+ // Check versions in delete by id
+ for (Map<String, Object> idParams : request.getDeleteByIdMap().values()) {
+ if (idParams != null) {
+ idParams.put(UpdateRequest.VER, null);
+ assertNull("Delete still has version", idParams.get(UpdateRequest.VER));
+ }
+ }
+ }
+
+ @Test
+ public void TestMethod() {
+
+ }
+}
diff --git a/crossdc-consumer/src/test/java/org/apache/solr/crossdc/TestMessageProcessor.java b/crossdc-consumer/src/test/java/org/apache/solr/crossdc/TestMessageProcessor.java
index 34f44f9..6158fc4 100644
--- a/crossdc-consumer/src/test/java/org/apache/solr/crossdc/TestMessageProcessor.java
+++ b/crossdc-consumer/src/test/java/org/apache/solr/crossdc/TestMessageProcessor.java
@@ -40,7 +40,7 @@ import static org.mockito.Mockito.*;
public class TestMessageProcessor {
static final String VERSION_FIELD = "_version_";
- private static class NoOpResubmitBackoffPolicy implements ResubmitBackoffPolicy {
+ static class NoOpResubmitBackoffPolicy implements ResubmitBackoffPolicy {
@Override
public long getBackoffTimeMs(MirroredSolrRequest resubmitRequest) {
return 0;