You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2017/03/19 15:57:38 UTC
[04/12] lucene-solr:jira/solr-6736: Merge master into jira/solr-6736
branch
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/schema.xml
new file mode 100644
index 0000000..31802f9
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/schema.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="minimal" version="1.1">
+
+ <field name="inplace_updatable_int" type="int" indexed="false" stored="false" docValues="true" />
+ <dynamicField name="*" type="string" indexed="true" stored="true"/>
+
+ <!-- for versioning -->
+ <field name="_version_" type="long" indexed="false" stored="false" docValues="true" />
+ <field name="id" type="string" indexed="true" stored="true" docValues="true"/>
+ <uniqueKey>id</uniqueKey>
+
+ <fieldType name="string" class="solr.StrField"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/solrconfig.xml
new file mode 100644
index 0000000..8da7d28
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-inplace-updates/conf/solrconfig.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ <updateLog class="${solr.ulog:solr.UpdateLog}"></updateLog>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
index 582c8b4..5eb4b3b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
@@ -54,11 +54,17 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
private static final String SHARD2 = "shard2";
private static final String SHARD1 = "shard1";
private static final String ONE_NODE_COLLECTION = "onenodecollection";
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
public BasicDistributedZk2Test() {
super();
sliceCount = 2;
}
+
+ @Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1 : -1;
+ }
@Test
@ShardsFixed(num = 4)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
index 25c483b..d1dbe9c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
@@ -87,6 +87,8 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String DEFAULT_COLLECTION = "collection1";
+
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
String t1="a_t";
String i1="a_i1";
String tlong = "other_tl1";
@@ -114,7 +116,12 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
pending = new HashSet<>();
}
-
+
+ @Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1 : -1;
+ }
+
@Override
protected void setDistributedParams(ModifiableSolrParams params) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
index 4e6122e..628884c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
@@ -55,6 +55,8 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
@BeforeClass
public static void beforeSuperClass() {
schemaString = "schema15.xml"; // we need a string id
@@ -109,6 +111,11 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
clientSoTimeout = 5000;
}
+ @Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1 : -1;
+ }
+
@Test
public void test() throws Exception {
cloudClient.setSoTimeout(clientSoTimeout);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
index e9e8907..8904ea8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
@@ -55,6 +55,12 @@ import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
public class ForceLeaderTest extends HttpPartitionTest {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
+ @Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1 : -1;
+ }
@Test
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
index 5ae4c17..01002cf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
@@ -76,12 +76,19 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
// give plenty of time for replicas to recover when running in slow Jenkins test envs
protected static final int maxWaitSecsToSeeAllActive = 90;
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
public HttpPartitionTest() {
super();
sliceCount = 2;
fixShardCount(3);
}
+ @Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1 : -1;
+ }
+
/**
* We need to turn off directUpdatesToLeadersOnly due to SOLR-9512
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
index fd122ad..457b9d9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java
@@ -37,6 +37,8 @@ public class LeaderInitiatedRecoveryOnCommitTest extends BasicDistributedZkTest
private static final long sleepMsBeforeHealPartition = 2000L;
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
+
public LeaderInitiatedRecoveryOnCommitTest() {
super();
sliceCount = 1;
@@ -44,6 +46,11 @@ public class LeaderInitiatedRecoveryOnCommitTest extends BasicDistributedZkTest
}
@Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1 : -1;
+ }
+
+ @Override
@Test
public void test() throws Exception {
oneShardTest();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java b/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
new file mode 100644
index 0000000..a4e8d6f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.update.SolrIndexWriter;
+import org.apache.solr.update.UpdateHandler;
+import org.apache.solr.update.UpdateLog;
+import org.apache.solr.util.RefCounted;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class OnlyLeaderIndexesTest extends SolrCloudTestCase {
+ private static final String COLLECTION = "collection1";
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
+ System.setProperty("solr.ulog.numRecordsToKeep", "1000");
+
+ configureCluster(3)
+ .addConfig("config", TEST_PATH().resolve("configsets")
+ .resolve("cloud-minimal-inplace-updates").resolve("conf"))
+ .configure();
+
+ CollectionAdminRequest
+ .createCollection(COLLECTION, "config", 1, 3)
+ .setRealtimeReplicas(1)
+ .setMaxShardsPerNode(1)
+ .process(cluster.getSolrClient());
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ }
+
+ @Test
+ public void test() throws Exception {
+ basicTest();
+ recoveryTest();
+ dbiTest();
+ basicLeaderElectionTest();
+ outOfOrderDBQWithInPlaceUpdatesTest();
+ }
+
+ public void basicTest() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ new UpdateRequest()
+ .add(sdoc("id", "1"))
+ .add(sdoc("id", "2"))
+ .add(sdoc("id", "3"))
+ .add(sdoc("id", "4"))
+ .process(cloudClient, COLLECTION);
+
+ {
+ UpdateHandler updateHandler = getSolrCore(true).get(0).getUpdateHandler();
+ RefCounted<IndexWriter> iwRef = updateHandler.getSolrCoreState().getIndexWriter(null);
+ assertTrue("IndexWriter at leader must see updates ", iwRef.get().hasUncommittedChanges());
+ iwRef.decref();
+ }
+
+ for (SolrCore solrCore : getSolrCore(false)) {
+ RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
+ assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
+ iwRef.decref();
+ }
+
+ checkRTG(1, 4, cluster.getJettySolrRunners());
+
+ new UpdateRequest()
+ .deleteById("1")
+ .deleteByQuery("id:2")
+ .process(cloudClient, COLLECTION);
+
+ // The DBQ is not processed at replicas, so we still can get doc2 and other docs by RTG
+ checkRTG(2,4, getSolrRunner(false));
+
+ new UpdateRequest()
+ .commit(cloudClient, COLLECTION);
+
+ checkShardConsistency(2, 1);
+
+ // Update log roll over
+ for (SolrCore solrCore : getSolrCore(false)) {
+ UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
+ assertFalse(updateLog.hasUncommittedChanges());
+ }
+
+ // UpdateLog copy over old updates
+ for (int i = 15; i <= 150; i++) {
+ cloudClient.add(COLLECTION, sdoc("id",String.valueOf(i)));
+ if (random().nextInt(100) < 15 & i != 150) {
+ cloudClient.commit(COLLECTION);
+ }
+ }
+ checkRTG(120,150, cluster.getJettySolrRunners());
+ waitForReplicasCatchUp(20);
+ }
+
+ public void recoveryTest() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTION);
+ new UpdateRequest()
+ .add(sdoc("id", "3"))
+ .add(sdoc("id", "4"))
+ .commit(cloudClient, COLLECTION);
+ // Replica recovery
+ new UpdateRequest()
+ .add(sdoc("id", "5"))
+ .process(cloudClient, COLLECTION);
+ JettySolrRunner solrRunner = getSolrRunner(false).get(0);
+ ChaosMonkey.stop(solrRunner);
+ new UpdateRequest()
+ .add(sdoc("id", "6"))
+ .process(cloudClient, COLLECTION);
+ ChaosMonkey.start(solrRunner);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ // We skip peerSync, so replica will always trigger commit on leader
+ checkShardConsistency(4, 20);
+
+ // LTR can be kicked off, so waiting for replicas recovery
+ new UpdateRequest()
+ .add(sdoc("id", "7"))
+ .commit(cloudClient, COLLECTION);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ checkShardConsistency(5, 20);
+
+ // More Replica recovery testing
+ new UpdateRequest()
+ .add(sdoc("id", "8"))
+ .process(cloudClient, COLLECTION);
+ checkRTG(3,8, cluster.getJettySolrRunners());
+ DirectUpdateHandler2.commitOnClose = false;
+ ChaosMonkey.stop(solrRunner);
+ DirectUpdateHandler2.commitOnClose = true;
+ ChaosMonkey.start(solrRunner);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ checkRTG(3,8, cluster.getJettySolrRunners());
+ checkShardConsistency(6, 20);
+
+ // Test replica recovery apply buffer updates
+ Semaphore waitingForBufferUpdates = new Semaphore(0);
+ Semaphore waitingForReplay = new Semaphore(0);
+ RecoveryStrategy.testing_beforeReplayBufferingUpdates = () -> {
+ try {
+ waitingForReplay.release();
+ waitingForBufferUpdates.acquire();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ };
+ ChaosMonkey.stop(solrRunner);
+ ChaosMonkey.start(solrRunner);
+ waitingForReplay.acquire();
+ new UpdateRequest()
+ .add(sdoc("id", "9"))
+ .add(sdoc("id", "10"))
+ .process(cloudClient, COLLECTION);
+ waitingForBufferUpdates.release();
+ RecoveryStrategy.testing_beforeReplayBufferingUpdates = null;
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ checkRTG(3,10, cluster.getJettySolrRunners());
+ checkShardConsistency(6, 20);
+ for (SolrCore solrCore : getSolrCore(false)) {
+ RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
+ assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
+ iwRef.decref();
+ }
+ }
+
+ public void dbiTest() throws Exception{
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTION);
+ new UpdateRequest()
+ .add(sdoc("id", "1"))
+ .commit(cloudClient, COLLECTION);
+ checkShardConsistency(1, 1);
+ new UpdateRequest()
+ .deleteById("1")
+ .process(cloudClient, COLLECTION);
+ try {
+ checkRTG(1, 1, cluster.getJettySolrRunners());
+ } catch (AssertionError e) {
+ return;
+ }
+ fail("Doc1 is deleted but it's still exist");
+ }
+
+ public void basicLeaderElectionTest() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTION);
+ new UpdateRequest()
+ .add(sdoc("id", "1"))
+ .add(sdoc("id", "2"))
+ .process(cloudClient, COLLECTION);
+ String oldLeader = getLeader();
+ JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
+ ChaosMonkey.kill(oldLeaderJetty);
+ for (int i = 0; i < 60; i++) { // wait till leader is changed
+ if (!oldLeader.equals(getLeader())) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+ new UpdateRequest()
+ .add(sdoc("id", "3"))
+ .add(sdoc("id", "4"))
+ .process(cloudClient, COLLECTION);
+ ChaosMonkey.start(oldLeaderJetty);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 60);
+ checkRTG(1,4, cluster.getJettySolrRunners());
+ new UpdateRequest()
+ .commit(cloudClient, COLLECTION);
+ checkShardConsistency(4,1);
+ }
+
+ private String getLeader() throws InterruptedException {
+ ZkNodeProps props = cluster.getSolrClient().getZkStateReader().getLeaderRetry("collection1", "shard1", 30000);
+ return props.getStr(ZkStateReader.NODE_NAME_PROP);
+ }
+
+ public void outOfOrderDBQWithInPlaceUpdatesTest() throws Exception {
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), COLLECTION);
+ List<UpdateRequest> updates = new ArrayList<>();
+ updates.add(simulatedUpdateRequest(null, "id", 1, "title_s", "title0_new", "inplace_updatable_int", 5, "_version_", Long.MAX_VALUE-100)); // full update
+ updates.add(simulatedDBQ("inplace_updatable_int:5", Long.MAX_VALUE-98));
+ updates.add(simulatedUpdateRequest(Long.MAX_VALUE-100, "id", 1, "inplace_updatable_int", 6, "_version_", Long.MAX_VALUE-99));
+ for (JettySolrRunner solrRunner: getSolrRunner(false)) {
+ try (SolrClient client = solrRunner.newClient()) {
+ for (UpdateRequest up : updates) {
+ up.process(client, COLLECTION);
+ }
+ }
+ }
+ JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
+ ChaosMonkey.kill(oldLeaderJetty);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ ChaosMonkey.start(oldLeaderJetty);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, 30);
+ new UpdateRequest()
+ .add(sdoc("id", "2"))
+ .commit(cluster.getSolrClient(), COLLECTION);
+ checkShardConsistency(2,20);
+ SolrDocument doc = cluster.getSolrClient().getById(COLLECTION,"1");
+ assertNotNull(doc.get("title_s"));
+ }
+
+ private UpdateRequest simulatedUpdateRequest(Long prevVersion, Object... fields) throws SolrServerException, IOException {
+ SolrInputDocument doc = sdoc(fields);
+
+ // get baseUrl of the leader
+ String baseUrl = getBaseUrl();
+
+ UpdateRequest ur = new UpdateRequest();
+ ur.add(doc);
+ ur.setParam("update.distrib", "FROMLEADER");
+ if (prevVersion != null) {
+ ur.setParam("distrib.inplace.prevversion", String.valueOf(prevVersion));
+ ur.setParam("distrib.inplace.update", "true");
+ }
+ ur.setParam("distrib.from", baseUrl);
+ return ur;
+ }
+
+ private UpdateRequest simulatedDBQ(String query, long version) throws SolrServerException, IOException {
+ String baseUrl = getBaseUrl();
+
+ UpdateRequest ur = new UpdateRequest();
+ ur.deleteByQuery(query);
+ ur.setParam("_version_", ""+version);
+ ur.setParam("update.distrib", "FROMLEADER");
+ ur.setParam("distrib.from", baseUrl);
+ return ur;
+ }
+
+ private String getBaseUrl() {
+ DocCollection collection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION);
+ Slice slice = collection.getSlice("shard1");
+ return slice.getLeader().getCoreUrl();
+ }
+
+ private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception{
+
+ for (JettySolrRunner solrRunner: solrRunners) {
+ try (SolrClient client = solrRunner.newClient()) {
+ for (int i = from; i <= to; i++) {
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ query.setRequestHandler("/get");
+ query.set("id",i);
+ QueryResponse res = client.query(COLLECTION, query);
+ assertNotNull("Can not find doc "+ i + " in " + solrRunner.getBaseUrl(),res.getResponse().get("doc"));
+ }
+ }
+ }
+
+ }
+
+ private void checkShardConsistency(int expected, int numTry) throws Exception{
+
+ for (int i = 0; i < numTry; i++) {
+ boolean inSync = true;
+ for (JettySolrRunner solrRunner: cluster.getJettySolrRunners()) {
+ try (SolrClient client = solrRunner.newClient()) {
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ long results = client.query(COLLECTION, query).getResults().getNumFound();
+ if (expected != results) {
+ inSync = false;
+ Thread.sleep(500);
+ break;
+ }
+ }
+ }
+ if (inSync) return;
+ }
+
+ fail("Some replicas are not in sync with leader");
+ }
+
+ private void waitForReplicasCatchUp(int numTry) throws IOException, InterruptedException {
+ String leaderTimeCommit = getSolrCore(true).get(0).getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
+ if (leaderTimeCommit == null) return;
+ for (int i = 0; i < numTry; i++) {
+ boolean inSync = true;
+ for (SolrCore solrCore : getSolrCore(false)) {
+ String replicateTimeCommit = solrCore.getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
+ if (!leaderTimeCommit.equals(replicateTimeCommit)) {
+ inSync = false;
+ Thread.sleep(500);
+ break;
+ }
+ }
+ if (inSync) return;
+ }
+
+ fail("Some replicas are not in sync with leader");
+
+ }
+
+ private List<SolrCore> getSolrCore(boolean isLeader) {
+ List<SolrCore> rs = new ArrayList<>();
+
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION);
+
+ for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+ if (solrRunner.getCoreContainer() == null) continue;
+ for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+ CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
+ Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
+ Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
+ if (slice.getLeader() == replica && isLeader) {
+ rs.add(solrCore);
+ } else if (slice.getLeader() != replica && !isLeader) {
+ rs.add(solrCore);
+ }
+ }
+ }
+ return rs;
+ }
+
+ private List<JettySolrRunner> getSolrRunner(boolean isLeader) {
+ List<JettySolrRunner> rs = new ArrayList<>();
+
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION);
+
+ for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
+ if (solrRunner.getCoreContainer() == null) continue;
+ for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
+ CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
+ Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
+ Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
+ if (slice.getLeader() == replica && isLeader) {
+ rs.add(solrRunner);
+ } else if (slice.getLeader() != replica && !isLeader) {
+ rs.add(solrRunner);
+ }
+ }
+ }
+ return rs;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/RecoveryAfterSoftCommitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryAfterSoftCommitTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryAfterSoftCommitTest.java
index c987c90..a8e14bf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/RecoveryAfterSoftCommitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryAfterSoftCommitTest.java
@@ -33,12 +33,17 @@ import org.junit.Test;
@SolrTestCaseJ4.SuppressSSL
public class RecoveryAfterSoftCommitTest extends AbstractFullDistribZkTestBase {
private static final int MAX_BUFFERED_DOCS = 2, ULOG_NUM_RECORDS_TO_KEEP = 2;
-
+ private final boolean onlyLeaderIndexes = random().nextBoolean();
public RecoveryAfterSoftCommitTest() {
sliceCount = 1;
fixShardCount(2);
}
+ @Override
+ protected int getRealtimeReplicas() {
+ return onlyLeaderIndexes? 1: -1;
+ }
+
@BeforeClass
public static void beforeTests() {
System.setProperty("solr.tests.maxBufferedDocs", String.valueOf(MAX_BUFFERED_DOCS));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
index 72f0694..bf9b5e0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
@@ -86,6 +86,12 @@ public class ShardSplitTest extends BasicDistributedZkTest {
useFactory(null);
}
+ //TODO for now, onlyLeaderIndexes do not work with ShardSplitTest
+ @Override
+ protected int getRealtimeReplicas() {
+ return -1;
+ }
+
@Test
public void test() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
index 18503e7..9c345fd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
@@ -38,6 +38,7 @@ import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
@@ -103,6 +104,11 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
public void setUp() throws Exception {
super.setUp();
collectionUlogDirMap.clear();
+ if (random().nextBoolean()) {
+ CollectionAdminRequest.setClusterProperty("legacyCloud", "false").process(cloudClient);
+ } else {
+ CollectionAdminRequest.setClusterProperty("legacyCloud", "true").process(cloudClient);
+ }
}
@Override
@@ -313,6 +319,29 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
assertSliceAndReplicaCount(collection1);
assertUlogDir(collections);
+
+ // restart all to test core saved state
+
+ ChaosMonkey.stop(jettys);
+ ChaosMonkey.stop(controlJetty);
+
+ assertTrue("Timeout waiting for all not live", ClusterStateUtil.waitForAllReplicasNotLive(cloudClient.getZkStateReader(), 45000));
+
+ ChaosMonkey.start(jettys);
+ ChaosMonkey.start(controlJetty);
+
+ assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cloudClient.getZkStateReader(), collection1, 120000));
+
+ assertSliceAndReplicaCount(collection1);
+
+ assertUlogDir(collections);
+
+ assertSliceAndReplicaCount(collection1);
+ assertSingleReplicationAndShardSize(collection3, 5);
+
+ // all docs should be queried
+ assertSingleReplicationAndShardSize(collection4, 5);
+ queryAndAssertResultSize(collection4, numDocs, 10000);
}
private void queryAndAssertResultSize(String collection, int expectedResultSize, int timeoutMS)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
index 1af09f4..b592861 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
@@ -52,6 +52,7 @@ import org.junit.Test;
public class TestCloudRecovery extends SolrCloudTestCase {
private static final String COLLECTION = "collection1";
+ private static boolean onlyLeaderIndexes;
@BeforeClass
public static void setupCluster() throws Exception {
@@ -63,8 +64,10 @@ public class TestCloudRecovery extends SolrCloudTestCase {
.addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
+ onlyLeaderIndexes = random().nextBoolean();
CollectionAdminRequest
.createCollection(COLLECTION, "config", 2, 2)
+ .setRealtimeReplicas(onlyLeaderIndexes? 1: -1)
.setMaxShardsPerNode(2)
.process(cluster.getSolrClient());
AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
@@ -107,7 +110,12 @@ public class TestCloudRecovery extends SolrCloudTestCase {
resp = cloudClient.query(COLLECTION, params);
assertEquals(4, resp.getResults().getNumFound());
// Make sure all nodes is recover from tlog
- assertEquals(4, countReplayLog.get());
+ if (onlyLeaderIndexes) {
+ // Leader election can be kicked off, so 2 append replicas will replay its tlog before becoming new leader
+ assertTrue( countReplayLog.get() >=2);
+ } else {
+ assertEquals(4, countReplayLog.get());
+ }
// check metrics
int replicationCount = 0;
@@ -127,7 +135,11 @@ public class TestCloudRecovery extends SolrCloudTestCase {
skippedCount += skipped.getCount();
}
}
- assertEquals(2, replicationCount);
+ if (onlyLeaderIndexes) {
+ assertTrue(replicationCount >= 2);
+ } else {
+ assertEquals(2, replicationCount);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
index 8905077..8fbfee3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
@@ -60,7 +60,10 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
@ShardsFixed(num = 2)
public void test() throws Exception {
try (CloudSolrClient client = createCloudClient(null)) {
- createCollection(null, COLLECTION_NAME, 2, 2, 2, client, null, "conf1");
+ CollectionAdminRequest.Create req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2,2);
+ req.setRealtimeReplicas(1);
+ req.setMaxShardsPerNode(2);
+ client.request(req);
createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
}
@@ -170,6 +173,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
Map<String, Object> collection = (Map<String, Object>) collections.get(COLLECTION_NAME);
assertNotNull(collection);
assertEquals("conf1", collection.get("configName"));
+ assertEquals("1", collection.get("realtimeReplicas"));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
index 966d8ef..140fd7e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
@@ -160,7 +160,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {
}
/**
- * Tests thta all TransformerFactories that are implicitly provided by Solr are tested in this class
+ * Tests that all TransformerFactories that are implicitly provided by Solr are tested in this class
*
* @see FlValidator#getDefaultTransformerFactoryName
* @see #FL_VALIDATORS
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
index 97be823..1bba523 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
@@ -42,7 +42,12 @@ public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest {
System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
+ @Override
+ protected int getRealtimeReplicas() {
+ return -1;
+ }
+
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/core/ConfigureRecoveryStrategyTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/ConfigureRecoveryStrategyTest.java b/solr/core/src/test/org/apache/solr/core/ConfigureRecoveryStrategyTest.java
new file mode 100644
index 0000000..0a988f6
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/ConfigureRecoveryStrategyTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cloud.RecoveryStrategy;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.junit.BeforeClass;
+
+/**
+ * test that configs can override the RecoveryStrategy
+ */
+public class ConfigureRecoveryStrategyTest extends SolrTestCaseJ4 {
+
+ private static final String solrConfigFileNameConfigure = "solrconfig-configurerecoverystrategy.xml";
+ private static final String solrConfigFileNameCustom = "solrconfig-customrecoverystrategy.xml";
+
+ private static String solrConfigFileName;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ solrConfigFileName = (random().nextBoolean()
+ ? solrConfigFileNameConfigure : solrConfigFileNameCustom);
+ initCore(solrConfigFileName, "schema.xml");
+ }
+
+ public void testBuilder() throws Exception {
+ final RecoveryStrategy.Builder recoveryStrategyBuilder =
+ h.getCore().getSolrCoreState().getRecoveryStrategyBuilder();
+ assertNotNull("recoveryStrategyBuilder is null", recoveryStrategyBuilder);
+
+ final String expectedClassName;
+
+ if (solrConfigFileName.equals(solrConfigFileNameConfigure)) {
+ expectedClassName = RecoveryStrategy.Builder.class.getName();
+ } else if (solrConfigFileName.equals(solrConfigFileNameCustom)) {
+ assertTrue("recoveryStrategyBuilder is wrong class (instanceof)",
+ recoveryStrategyBuilder instanceof CustomRecoveryStrategyBuilder);
+ expectedClassName = ConfigureRecoveryStrategyTest.CustomRecoveryStrategyBuilder.class.getName();
+ } else {
+ expectedClassName = null;
+ }
+
+ assertEquals("recoveryStrategyBuilder is wrong class (name)",
+ expectedClassName, recoveryStrategyBuilder.getClass().getName());
+ }
+
+ public void testAlmostAllMethodsAreFinal() throws Exception {
+ for (Method m : RecoveryStrategy.class.getDeclaredMethods()) {
+ if (Modifier.isStatic(m.getModifiers())) continue;
+ final String methodName = m.getName();
+ if ("getReplicateLeaderUrl".equals(methodName)) {
+ assertFalse(m.toString(), Modifier.isFinal(m.getModifiers()));
+ } else {
+ assertTrue(m.toString(), Modifier.isFinal(m.getModifiers()));
+ }
+ }
+ }
+
+ static public class CustomRecoveryStrategy extends RecoveryStrategy {
+
+ private String alternativeBaseUrlProp;
+
+ public String getAlternativeBaseUrlProp() {
+ return alternativeBaseUrlProp;
+ }
+
+ public void setAlternativeBaseUrlProp(String alternativeBaseUrlProp) {
+ this.alternativeBaseUrlProp = alternativeBaseUrlProp;
+ }
+
+ public CustomRecoveryStrategy(CoreContainer cc, CoreDescriptor cd,
+ RecoveryStrategy.RecoveryListener recoveryListener) {
+ super(cc, cd, recoveryListener);
+ }
+
+ @Override
+ protected String getReplicateLeaderUrl(ZkNodeProps leaderprops) {
+ return ZkCoreNodeProps.getCoreUrl(
+ leaderprops.getStr(alternativeBaseUrlProp),
+ leaderprops.getStr(ZkStateReader.CORE_NAME_PROP));
+ }
+ }
+
+ static public class CustomRecoveryStrategyBuilder extends RecoveryStrategy.Builder {
+ @Override
+ protected RecoveryStrategy newRecoveryStrategy(CoreContainer cc, CoreDescriptor cd,
+ RecoveryStrategy.RecoveryListener recoveryListener) {
+ return new CustomRecoveryStrategy(cc, cd, recoveryListener);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
index bb56a94..9503ee4 100644
--- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
+++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
@@ -295,7 +295,7 @@ public class TestSolrCloudSnapshots extends SolrCloudTestCase {
for(int i = 0 ; i < apiResult.size(); i++) {
String commitName = apiResult.getName(i);
String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.INDEX_DIR_PATH);
- long genNumber = Long.valueOf((String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.GENERATION_NUM));
+ long genNumber = Long.parseLong((String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.GENERATION_NUM));
result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber));
}
return result;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
index da6dbac..7a9b0bb 100644
--- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
+++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
@@ -293,7 +293,7 @@ public class TestSolrCoreSnapshots extends SolrCloudTestCase {
for(int i = 0 ; i < apiResult.size(); i++) {
String commitName = apiResult.getName(i);
String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get("indexDirPath");
- long genNumber = Long.valueOf((String)((NamedList)apiResult.get(commitName)).get("generation"));
+ long genNumber = Long.parseLong((String)((NamedList)apiResult.get(commitName)).get("generation"));
result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber));
}
return result;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
index f222cee..cb16f03 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
@@ -16,21 +16,30 @@
*/
package org.apache.solr.handler;
+import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.stream.ExceptionStream;
import org.apache.solr.client.solrj.io.stream.SolrStream;
import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -161,6 +170,9 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
assert(tuple.getLong("field_i") == 7);
assert(tuple.get("str_s").equals("a"));
+ //Assert field order
+ assertResponseContains(clients.get(0), sParams, "{\"docs\":[{\"id\":8,\"field_i\":60,\"str_s\":\"c\"}");
+
//Test unlimited unsorted result. Should sort on _version_ desc
sParams = mapParams(CommonParams.QT, "/sql", "stmt", "select id, field_i, str_s from collection1 where text='XXXX'");
@@ -2362,4 +2374,23 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
return params;
}
+ public void assertResponseContains(SolrClient server, SolrParams requestParams, String json) throws IOException, SolrServerException {
+ String p = requestParams.get("qt");
+ if(p != null) {
+ ModifiableSolrParams modifiableSolrParams = (ModifiableSolrParams) requestParams;
+ modifiableSolrParams.remove("qt");
+ }
+
+ QueryRequest query = new QueryRequest( requestParams );
+ query.setPath(p);
+ query.setResponseParser(new InputStreamResponseParser("json"));
+ query.setMethod(SolrRequest.METHOD.POST);
+ NamedList<Object> genericResponse = server.request(query);
+ InputStream stream = (InputStream)genericResponse.get("stream");
+ InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ String response = bufferedReader.readLine();
+ assertTrue(response.contains(json));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index a1b29db..81e14d9 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -17,6 +17,8 @@
package org.apache.solr.handler.admin;
+import java.util.Map;
+
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
@@ -47,14 +49,17 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(values.get("solr.node"));
NamedList nl = (NamedList) values.get("solr.core.collection1");
assertNotNull(nl);
- assertNotNull(nl.get("SEARCHER.new.errors")); // counter type
- assertNotNull(((NamedList) nl.get("SEARCHER.new.errors")).get("count"));
- assertEquals(0L, ((NamedList) nl.get("SEARCHER.new.errors")).get("count"));
+ Object o = nl.get("SEARCHER.new.errors");
+ assertNotNull(o); // counter type
+ assertTrue(o instanceof Map);
+ // response wasn't serialized so we get here whatever MetricUtils produced instead of NamedList
+ assertNotNull(((Map) o).get("count"));
+ assertEquals(0L, ((Map) nl.get("SEARCHER.new.errors")).get("count"));
nl = (NamedList) values.get("solr.node");
assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge
- assertEquals(1, ((NamedList) nl.get("CONTAINER.cores.loaded")).get("value"));
+ assertEquals(1, ((Map) nl.get("CONTAINER.cores.loaded")).get("value"));
assertNotNull(nl.get("ADMIN./admin/authorization.clientErrors")); // timer type
- assertEquals(5, ((NamedList) nl.get("ADMIN./admin/authorization.clientErrors")).size());
+ assertEquals(5, ((Map) nl.get("ADMIN./admin/authorization.clientErrors")).size());
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", "group", "jvm,jetty"), resp);
@@ -146,4 +151,20 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertEquals(1, values.size());
assertEquals(0, ((NamedList)values.get("solr.node")).size());
}
+
+ @Test
+ public void testCompact() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ SolrQueryResponse resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", MetricsHandler.COMPACT_PARAM, "true"), resp);
+ NamedList values = resp.getValues();
+ assertNotNull(values.get("metrics"));
+ values = (NamedList) values.get("metrics");
+ NamedList nl = (NamedList) values.get("solr.core.collection1");
+ assertNotNull(nl);
+ Object o = nl.get("SEARCHER.new.errors");
+ assertNotNull(o); // counter type
+ assertTrue(o instanceof Number);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java
new file mode 100644
index 0000000..1a959a4
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/PropertiesRequestHandlerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import java.io.StringReader;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.RedactionUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class PropertiesRequestHandlerTest extends SolrTestCaseJ4 {
+
+ public static final String PASSWORD = "secret123";
+ public static final String REDACT_STRING = RedactionUtils.getRedactString();
+
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig.xml", "schema.xml");
+ }
+
+ @Test
+ public void testRedaction() throws Exception {
+ RedactionUtils.setRedactSystemProperty(true);
+ for(String propName: new String[]{"some.password", "javax.net.ssl.trustStorePassword"}){
+ System.setProperty(propName, PASSWORD);
+ NamedList<NamedList<NamedList<Object>>> properties = readProperties();
+
+ assertEquals("Failed to redact "+propName, REDACT_STRING, properties.get(propName));
+ }
+ }
+
+ @Test
+ public void testDisabledRedaction() throws Exception {
+ RedactionUtils.setRedactSystemProperty(false);
+ for(String propName: new String[]{"some.password", "javax.net.ssl.trustStorePassword"}){
+ System.setProperty(propName, PASSWORD);
+ NamedList<NamedList<NamedList<Object>>> properties = readProperties();
+
+ assertEquals("Failed to *not* redact "+propName, PASSWORD, properties.get(propName));
+ }
+ }
+
+ private NamedList<NamedList<NamedList<Object>>> readProperties() throws Exception {
+ String xml = h.query(req(
+ CommonParams.QT, "/admin/properties",
+ CommonParams.WT, "xml"
+ ));
+
+ XMLResponseParser parser = new XMLResponseParser();
+ return (NamedList<NamedList<NamedList<Object>>>)
+ parser.processResponse(new StringReader(xml)).get("system.properties");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java b/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
index 9835518..2f7a003 100644
--- a/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
+++ b/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
@@ -19,6 +19,7 @@ package org.apache.solr.highlight;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
/** Tests for the UnifiedHighlighter Solr plugin **/
@@ -26,7 +27,10 @@ public class TestUnifiedSolrHighlighter extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-basic.xml", "schema-unifiedhighlight.xml");
+ System.setProperty("filterCache.enabled", "false");
+ System.setProperty("queryResultCache.enabled", "false");
+ System.setProperty("documentCache.enabled", "true"); // this is why we use this particular solrconfig
+ initCore("solrconfig-cache-enable-disable.xml", "schema-unifiedhighlight.xml");
// test our config is sane, just to be sure:
@@ -36,6 +40,12 @@ public class TestUnifiedSolrHighlighter extends SolrTestCaseJ4 {
assertTrue(schema.getField("text3").storeOffsetsWithPositions());
assertFalse(schema.getField("text2").storeOffsetsWithPositions());
}
+ @AfterClass
+ public static void afterClass() {
+ System.clearProperty("filterCache.enabled");
+ System.clearProperty("queryResultCache.enabled");
+ System.clearProperty("documentCache.enabled");
+ }
@Override
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/index/UninvertDocValuesMergePolicyTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/index/UninvertDocValuesMergePolicyTest.java b/solr/core/src/test/org/apache/solr/index/UninvertDocValuesMergePolicyTest.java
new file mode 100644
index 0000000..17e6b3e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/index/UninvertDocValuesMergePolicyTest.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.index;
+
+import java.util.Random;
+import java.util.function.IntUnaryOperator;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
+import org.apache.solr.util.TestHarness;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+public class UninvertDocValuesMergePolicyTest extends SolrTestCaseJ4 {
+
+ private static String SOLR_TESTS_SKIP_INTEGRITY_CHECK = "solr.tests.skipIntegrityCheck";
+ private static String ID_FIELD = "id";
+ private static String TEST_FIELD = "string_add_dv_later";
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ System.setProperty(SOLR_TESTS_SKIP_INTEGRITY_CHECK, (random().nextBoolean() ? "true" : "false"));
+ }
+
+ @AfterClass
+ public static void afterTests() {
+ System.clearProperty(SOLR_TESTS_SKIP_INTEGRITY_CHECK);
+ }
+
+ @After
+ public void after() throws Exception {
+ deleteCore();
+ }
+
+ @Before
+ public void before() throws Exception {
+ initCore("solrconfig-uninvertdocvaluesmergepolicyfactory.xml", "schema-docValues.xml");
+ }
+
+ public void testIndexAndAddDocValues() throws Exception {
+ Random rand = random();
+
+ for(int i=0; i < 100; i++) {
+ assertU(adoc(ID_FIELD, String.valueOf(i), TEST_FIELD, String.valueOf(i)));
+
+ if(rand.nextBoolean()) {
+ assertU(commit());
+ }
+ }
+
+ assertU(commit());
+
+ // Assert everything has been indexed and there are no docvalues
+ withNewRawReader(h, topReader -> {
+ assertEquals(100, topReader.numDocs());
+
+ final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader);
+
+ // The global field type should not have docValues yet
+ assertEquals(DocValuesType.NONE, infos.fieldInfo(TEST_FIELD).getDocValuesType());
+ });
+
+
+ addDocValuesTo(h, TEST_FIELD);
+
+
+ // Add some more documents with doc values turned on including updating some
+ for(int i=90; i < 110; i++) {
+ assertU(adoc(ID_FIELD, String.valueOf(i), TEST_FIELD, String.valueOf(i)));
+
+ if(rand.nextBoolean()) {
+ assertU(commit());
+ }
+ }
+
+ assertU(commit());
+
+ withNewRawReader(h, topReader -> {
+ assertEquals(110, topReader.numDocs());
+
+ final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader);
+ // The global field type should have docValues because a document with dvs was added
+ assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType());
+ });
+
+ int optimizeSegments = 1;
+ assertU(optimize("maxSegments", String.valueOf(optimizeSegments)));
+
+
+ // Assert all docs have the right docvalues
+ withNewRawReader(h, topReader -> {
+ // Assert merged into one segment
+ assertEquals(110, topReader.numDocs());
+ assertEquals(optimizeSegments, topReader.leaves().size());
+
+
+ final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader);
+ // The global field type should have docValues because a document with dvs was added
+ assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType());
+
+
+ // Check that all segments have the right docvalues type with the correct value
+ // Also check that other fields (e.g. the id field) didn't mistakenly get docvalues added
+ for (LeafReaderContext ctx : topReader.leaves()) {
+ LeafReader r = ctx.reader();
+ SortedDocValues docvalues = r.getSortedDocValues(TEST_FIELD);
+ for(int i = 0; i < r.numDocs(); ++i) {
+ Document doc = r.document(i);
+ String v = doc.getField(TEST_FIELD).stringValue();
+ String id = doc.getField(ID_FIELD).stringValue();
+ assertEquals(DocValuesType.SORTED, r.getFieldInfos().fieldInfo(TEST_FIELD).getDocValuesType());
+ assertEquals(DocValuesType.NONE, r.getFieldInfos().fieldInfo(ID_FIELD).getDocValuesType());
+ assertEquals(v, id);
+
+ docvalues.nextDoc();
+ assertEquals(v, docvalues.binaryValue().utf8ToString());
+ }
+ }
+ });
+ }
+
+
+ // When an non-indexed field gets merged, it exhibit the old behavior
+ // The field will be merged, docvalues headers updated, but no docvalues for this field
+ public void testNonIndexedFieldDoesNonFail() throws Exception {
+ // Remove Indexed from fieldType
+ removeIndexFrom(h, TEST_FIELD);
+
+ assertU(adoc(ID_FIELD, String.valueOf(1), TEST_FIELD, String.valueOf(1)));
+ assertU(commit());
+
+ addDocValuesTo(h, TEST_FIELD);
+
+ assertU(adoc(ID_FIELD, String.valueOf(2), TEST_FIELD, String.valueOf(2)));
+ assertU(commit());
+
+ assertU(optimize("maxSegments", "1"));
+
+ withNewRawReader(h, topReader -> {
+ // Assert merged into one segment
+ assertEquals(2, topReader.numDocs());
+ assertEquals(1, topReader.leaves().size());
+
+
+ final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader);
+ // The global field type should have docValues because a document with dvs was added
+ assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType());
+
+ for (LeafReaderContext ctx : topReader.leaves()) {
+ LeafReader r = ctx.reader();
+ SortedDocValues docvalues = r.getSortedDocValues(TEST_FIELD);
+ for(int i = 0; i < r.numDocs(); ++i) {
+ Document doc = r.document(i);
+ String v = doc.getField(TEST_FIELD).stringValue();
+ String id = doc.getField(ID_FIELD).stringValue();
+ assertEquals(DocValuesType.SORTED, r.getFieldInfos().fieldInfo(TEST_FIELD).getDocValuesType());
+ assertEquals(DocValuesType.NONE, r.getFieldInfos().fieldInfo(ID_FIELD).getDocValuesType());
+
+
+ if(id.equals("2")) {
+ assertTrue(docvalues.advanceExact(i));
+ assertEquals(v, docvalues.binaryValue().utf8ToString());
+ } else {
+ assertFalse(docvalues.advanceExact(i));
+ }
+
+ }
+ }
+ });
+ }
+
+
+ private static void addDocValuesTo(TestHarness h, String fieldName) {
+ implUpdateSchemaField(h, fieldName, (p) -> (p | 0x00008000)); // FieldProperties.DOC_VALUES
+ }
+
+ private static void removeIndexFrom(TestHarness h, String fieldName) {
+ implUpdateSchemaField(h, fieldName, (p) -> (p ^ 0x00000001)); // FieldProperties.INDEXED
+ }
+
+ private static void implUpdateSchemaField(TestHarness h, String fieldName, IntUnaryOperator propertiesModifier) {
+ try (SolrCore core = h.getCoreInc()) {
+
+ // Add docvalues to the field type
+ IndexSchema schema = core.getLatestSchema();
+ SchemaField oldSchemaField = schema.getField(fieldName);
+ SchemaField newSchemaField = new SchemaField(
+ fieldName,
+ oldSchemaField.getType(),
+ propertiesModifier.applyAsInt(oldSchemaField.getProperties()),
+ oldSchemaField.getDefaultValue());
+ schema.getFields().put(fieldName, newSchemaField);
+ }
+ }
+
+ private interface DirectoryReaderConsumer {
+ public void accept(DirectoryReader consumer) throws Exception;
+ }
+
+ private static void withNewRawReader(TestHarness h, DirectoryReaderConsumer consumer) {
+ try (SolrCore core = h.getCoreInc()) {
+ final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true);
+ final SolrIndexSearcher searcher = searcherRef.get();
+ try {
+ try {
+ consumer.accept(searcher.getRawReader());
+ } catch (Exception e) {
+ fail(e.toString());
+ }
+ } finally {
+ searcherRef.decref();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java b/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
index e2dc2bf..166d1fc 100644
--- a/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
+++ b/solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
@@ -18,6 +18,7 @@ package org.apache.solr.rest.schema;
import org.apache.commons.io.FileUtils;
+import org.apache.lucene.search.similarities.DFISimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.misc.SweetSpotSimilarity;
@@ -42,10 +43,12 @@ import java.io.File;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Consumer;
public class TestBulkSchemaAPI extends RestTestBase {
@@ -798,6 +801,68 @@ public class TestBulkSchemaAPI extends RestTestBase {
assertNull(map.get("errors"));
}
+ public void testSimilarityParser() throws Exception {
+ RestTestHarness harness = restTestHarness;
+
+ final float k1 = 2.25f;
+ final float b = 0.33f;
+
+ String fieldTypeName = "MySimilarityField";
+ String fieldName = "similarityTestField";
+ String payload = "{\n" +
+ " 'add-field-type' : {" +
+ " 'name' : '" + fieldTypeName + "',\n" +
+ " 'class':'solr.TextField',\n" +
+ " 'analyzer' : {'tokenizer':{'class':'solr.WhitespaceTokenizerFactory'}},\n" +
+ " 'similarity' : {'class':'org.apache.solr.search.similarities.BM25SimilarityFactory', 'k1':"+k1+", 'b':"+b+" }\n" +
+ " },\n"+
+ " 'add-field' : {\n" +
+ " 'name':'" + fieldName + "',\n" +
+ " 'type': 'MySimilarityField',\n" +
+ " 'stored':true,\n" +
+ " 'indexed':true\n" +
+ " }\n" +
+ "}\n";
+
+ String response = harness.post("/schema?wt=json&indent=on", json(payload));
+
+ Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ assertNull(response, map.get("errors"));
+
+ Map fields = getObj(harness, fieldName, "fields");
+ assertNotNull("field " + fieldName + " not created", fields);
+
+ assertFieldSimilarity(fieldName, BM25Similarity.class,
+ sim -> assertEquals("Unexpected k1", k1, sim.getK1(), .001),
+ sim -> assertEquals("Unexpected b", b, sim.getB(), .001));
+
+ final String independenceMeasure = "Saturated";
+ final boolean discountOverlaps = false;
+ payload = "{\n" +
+ " 'replace-field-type' : {" +
+ " 'name' : '" + fieldTypeName + "',\n" +
+ " 'class':'solr.TextField',\n" +
+ " 'analyzer' : {'tokenizer':{'class':'solr.WhitespaceTokenizerFactory'}},\n" +
+ " 'similarity' : {\n" +
+ " 'class':'org.apache.solr.search.similarities.DFISimilarityFactory',\n" +
+ " 'independenceMeasure':'" + independenceMeasure + "',\n" +
+ " 'discountOverlaps':" + discountOverlaps + "\n" +
+ " }\n" +
+ " }\n"+
+ "}\n";
+
+ response = harness.post("/schema?wt=json&indent=on", json(payload));
+
+ map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ assertNull(response, map.get("errors"));
+ fields = getObj(harness, fieldName, "fields");
+ assertNotNull("field " + fieldName + " not created", fields);
+
+ assertFieldSimilarity(fieldName, DFISimilarity.class,
+ sim -> assertEquals("Unexpected independenceMeasure", independenceMeasure, sim.getIndependence().toString()),
+ sim -> assertEquals("Unexpected discountedOverlaps", discountOverlaps, sim.getDiscountOverlaps()));
+ }
+
public static Map getObj(RestTestHarness restHarness, String fld, String key) throws Exception {
Map map = getRespMap(restHarness);
List l = (List) ((Map)map.get("schema")).get(key);
@@ -842,8 +907,11 @@ public class TestBulkSchemaAPI extends RestTestBase {
/**
* whitebox checks the Similarity for the specified field according to {@link SolrCore#getLatestSchema}
+ *
+ * Executes each of the specified Similarity-accepting validators.
*/
- private static void assertFieldSimilarity(String fieldname, Class<? extends Similarity> expected) {
+ @SafeVarargs
+ private static <T extends Similarity> void assertFieldSimilarity(String fieldname, Class<T> expected, Consumer<T>... validators) {
CoreContainer cc = jetty.getCoreContainer();
try (SolrCore core = cc.getCore("collection1")) {
SimilarityFactory simfac = core.getLatestSchema().getSimilarityFactory();
@@ -861,7 +929,7 @@ public class TestBulkSchemaAPI extends RestTestBase {
mainSim instanceof PerFieldSimilarityWrapper);
Similarity fieldSim = ((PerFieldSimilarityWrapper)mainSim).get(fieldname);
assertEquals("wrong sim for field=" + fieldname, expected, fieldSim.getClass());
-
+ Arrays.asList(validators).forEach(v -> v.accept((T)fieldSim));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java b/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java
index 6839c70..900e439 100644
--- a/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java
@@ -15,6 +15,7 @@
* limitations under the License.
*/
package org.apache.solr.schema;
+import java.util.Arrays;
import java.util.List;
import org.apache.lucene.index.IndexableField;
@@ -45,11 +46,16 @@ public class PolyFieldTest extends SolrTestCaseJ4 {
SchemaField home = schema.getField("home");
assertNotNull(home);
assertTrue(home.isPolyField());
+
+ String subFieldType = "double";
+ if (usingPointFields()) {
+ subFieldType = "pdouble";
+ }
SchemaField[] dynFields = schema.getDynamicFieldPrototypes();
boolean seen = false;
for (SchemaField dynField : dynFields) {
- if (dynField.getName().equals("*" + FieldType.POLY_FIELD_SEPARATOR + "double")) {
+ if (dynField.getName().equals("*" + FieldType.POLY_FIELD_SEPARATOR + subFieldType)) {
seen = true;
}
}
@@ -60,7 +66,7 @@ public class PolyFieldTest extends SolrTestCaseJ4 {
assertNotNull(xy);
assertTrue(xy instanceof PointType);
assertTrue(xy.isPolyField());
- home = schema.getFieldOrNull("home_0" + FieldType.POLY_FIELD_SEPARATOR + "double");
+ home = schema.getFieldOrNull("home_0" + FieldType.POLY_FIELD_SEPARATOR + subFieldType);
assertNotNull(home);
home = schema.getField("home");
assertNotNull(home);
@@ -84,9 +90,14 @@ public class PolyFieldTest extends SolrTestCaseJ4 {
double[] xy = new double[]{35.0, -79.34};
String point = xy[0] + "," + xy[1];
List<IndexableField> fields = home.createFields(point);
- assertEquals(fields.size(), 3);//should be 3, we have a stored field
- //first two fields contain the values, third is just stored and contains the original
- for (int i = 0; i < 3; i++) {
+ assertNotNull(pt.getSubType());
+ int expectdNumFields = 3;//If DV=false, we expect one field per dimension plus a stored field
+ if (pt.subField(home, 0, schema).hasDocValues()) {
+ expectdNumFields+=2; // If docValues=true, then we expect two more fields
+ }
+ assertEquals("Unexpected fields created: " + Arrays.toString(fields.toArray()), expectdNumFields, fields.size());
+ //first two/four fields contain the values, last one is just stored and contains the original
+ for (int i = 0; i < expectdNumFields; i++) {
boolean hasValue = fields.get(i).binaryValue() != null
|| fields.get(i).stringValue() != null
|| fields.get(i).numericValue() != null;
@@ -100,7 +111,7 @@ public class PolyFieldTest extends SolrTestCaseJ4 {
home = schema.getField("home_ns");
assertNotNull(home);
fields = home.createFields(point);
- assertEquals(fields.size(), 2);//should be 2, since we aren't storing
+ assertEquals(expectdNumFields - 1, fields.size(), 2);//one less field than with "home", since we aren't storing
home = schema.getField("home_ns");
assertNotNull(home);
@@ -111,17 +122,12 @@ public class PolyFieldTest extends SolrTestCaseJ4 {
//
}
-
SchemaField s1 = schema.getField("test_p");
SchemaField s2 = schema.getField("test_p");
- // If we use [Int/Double/Long/Float]PointField, we can't get the valueSource, since docValues is false
- if (s1.createFields("1,2").get(0).fieldType().pointDimensionCount() == 0) {
- assertFalse(s2.getType().isPointField());
- ValueSource v1 = s1.getType().getValueSource(s1, null);
- ValueSource v2 = s2.getType().getValueSource(s2, null);
- assertEquals(v1, v2);
- assertEquals(v1.hashCode(), v2.hashCode());
- }
+ ValueSource v1 = s1.getType().getValueSource(s1, null);
+ ValueSource v2 = s2.getType().getValueSource(s2, null);
+ assertEquals(v1, v2);
+ assertEquals(v1.hashCode(), v2.hashCode());
}
@Test
@@ -181,5 +187,9 @@ public class PolyFieldTest extends SolrTestCaseJ4 {
assertEquals(2, bq.clauses().size());
clearIndex();
}
+
+ private boolean usingPointFields() {
+ return h.getCore().getLatestSchema().getField("foo_d1_dv").getType().isPointField();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/887d1b18/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java b/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
index b8ed296..90c92d0 100644
--- a/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
+++ b/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
@@ -35,8 +35,7 @@ import org.apache.lucene.util.BytesRef;
public class SortableBinaryField extends BinaryField {
@Override
- public void checkSchemaField(final SchemaField field) {
- // NOOP, It's Aaaaaall Good.
+ protected void checkSupportsDocValues() { // we support DocValues
}
@Override