You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2016/12/14 22:15:24 UTC
[31/32] lucene-solr:jira/solr-5944: SOLR-8959: Refactored
TestSegmentSorting out of TestMiniSolrCloudCluster
SOLR-8959: Refactored TestSegmentSorting out of TestMiniSolrCloudCluster
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6525bb56
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6525bb56
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6525bb56
Branch: refs/heads/jira/solr-5944
Commit: 6525bb56f027655e5a01f028fa373305c0d01caa
Parents: 5123743
Author: Chris Hostetter <ho...@apache.org>
Authored: Wed Dec 14 13:18:56 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Wed Dec 14 13:18:56 2016 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../cloud/SegmentTerminateEarlyTestState.java | 12 +-
.../solr/cloud/TestMiniSolrCloudCluster.java | 50 -------
.../apache/solr/cloud/TestSegmentSorting.java | 133 +++++++++++++++++++
4 files changed, 145 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6525bb56/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5f0357b..73b0e9b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -290,6 +290,9 @@ Other Changes
* SOLR-9846: OverseerAutoReplicaFailoverThread can take too long to stop and leak out of unit tests. (Mark Miller)
+* SOLR-8959: Refactored TestSegmentSorting out of TestMiniSolrCloudCluster (hossman)
+
+
================== 6.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6525bb56/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java b/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java
index 199423b..b3df9e7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java
@@ -22,6 +22,7 @@ import java.time.ZonedDateTime;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.Random;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -47,7 +48,12 @@ class SegmentTerminateEarlyTestState {
Integer maxTimestampMM = null;
int numDocs = 0;
+ final Random rand;
+ public SegmentTerminateEarlyTestState(Random rand) {
+ this.rand = rand;
+ }
+
void addDocuments(CloudSolrClient cloudSolrClient,
int numCommits, int numDocsPerCommit, boolean optimize) throws Exception {
for (int cc = 1; cc <= numCommits; ++cc) {
@@ -56,7 +62,7 @@ class SegmentTerminateEarlyTestState {
final Integer docKey = new Integer(numDocs);
SolrInputDocument doc = new SolrInputDocument();
doc.setField(keyField, ""+docKey);
- final int MM = TestMiniSolrCloudCluster.random().nextInt(60); // minutes
+ final int MM = rand.nextInt(60); // minutes
if (minTimestampMM == null || MM <= minTimestampMM.intValue()) {
if (minTimestampMM != null && MM < minTimestampMM.intValue()) {
minTimestampDocKeys.clear();
@@ -116,7 +122,7 @@ class SegmentTerminateEarlyTestState {
query.setFields(keyField, oddField, timestampField);
final int rowsWanted = 1;
query.setRows(rowsWanted);
- final Boolean shardsInfoWanted = (TestMiniSolrCloudCluster.random().nextBoolean() ? null : new Boolean(TestMiniSolrCloudCluster.random().nextBoolean()));
+ final Boolean shardsInfoWanted = (rand.nextBoolean() ? null : new Boolean(rand.nextBoolean()));
if (shardsInfoWanted != null) {
query.set(ShardParams.SHARDS_INFO, shardsInfoWanted.booleanValue());
}
@@ -163,7 +169,7 @@ class SegmentTerminateEarlyTestState {
query.setSort(timestampField, SolrQuery.ORDER.desc);
query.setFields(keyField, oddField, timestampField);
query.setRows(1);
- final Boolean shardsInfoWanted = (TestMiniSolrCloudCluster.random().nextBoolean() ? null : new Boolean(TestMiniSolrCloudCluster.random().nextBoolean()));
+ final Boolean shardsInfoWanted = (rand.nextBoolean() ? null : new Boolean(rand.nextBoolean()));
if (shardsInfoWanted != null) {
query.set(ShardParams.SHARDS_INFO, shardsInfoWanted.booleanValue());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6525bb56/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
index 97ecb67..de18875 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.cloud;
-import java.io.File;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.ArrayList;
@@ -384,53 +383,4 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase {
}
}
- @Test
- public void testSegmentTerminateEarly() throws Exception {
-
- final String collectionName = "testSegmentTerminateEarlyCollection";
-
- final SegmentTerminateEarlyTestState tstes = new SegmentTerminateEarlyTestState();
-
- File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml");
- Builder jettyConfig = JettyConfig.builder();
- jettyConfig.waitForLoadingCoresToFinish(null);
- final MiniSolrCloudCluster miniCluster = createMiniSolrCloudCluster();
- final CloudSolrClient cloudSolrClient = miniCluster.getSolrClient();
- cloudSolrClient.setDefaultCollection(collectionName);
-
- try {
- // create collection
- {
- final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
- final Map<String, String> collectionProperties = new HashMap<>();
- collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml");
- createCollection(miniCluster, collectionName, null, asyncId, Boolean.TRUE, collectionProperties);
- }
-
- ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
-
- // add some documents, then optimize to get merged-sorted segments
- tstes.addDocuments(cloudSolrClient, 10, 10, true);
-
- // CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent
- tstes.queryTimestampDescending(cloudSolrClient);
-
- // add a few more documents, but don't optimize to have some not-merge-sorted segments
- tstes.addDocuments(cloudSolrClient, 2, 10, false);
-
- // CommonParams.SEGMENT_TERMINATE_EARLY parameter now present
- tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient);
- tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient);
-
- // CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used
- tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient);
- tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order
-
- }
- finally {
- miniCluster.shutdown();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6525bb56/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
new file mode 100644
index 0000000..016b63e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.index.TieredMergePolicyFactory;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestSegmentSorting extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final int NUM_SERVERS = 5;
+ private static final int NUM_SHARDS = 2;
+ private static final int REPLICATION_FACTOR = 2;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(NUM_SERVERS).configure();
+ }
+
+ @After
+ public void ensureClusterEmpty() throws Exception {
+ cluster.deleteAllCollections();
+ cluster.getSolrClient().setDefaultCollection(null);
+ }
+
+ private void createCollection(MiniSolrCloudCluster miniCluster, String collectionName, String createNodeSet, String asyncId,
+ Boolean indexToPersist, Map<String,String> collectionProperties) throws Exception {
+ String configName = "solrCloudCollectionConfig";
+ miniCluster.uploadConfigSet(SolrTestCaseJ4.TEST_PATH().resolve("collection1").resolve("conf"), configName);
+
+ final boolean persistIndex = (indexToPersist != null ? indexToPersist.booleanValue() : random().nextBoolean());
+ if (collectionProperties == null) {
+ collectionProperties = new HashMap<>();
+ }
+ collectionProperties.putIfAbsent(CoreDescriptor.CORE_CONFIG, "solrconfig-tlog.xml");
+ collectionProperties.putIfAbsent("solr.tests.maxBufferedDocs", "100000");
+ collectionProperties.putIfAbsent("solr.tests.ramBufferSizeMB", "100");
+ // use non-test classes so RandomizedRunner isn't necessary
+ if (random().nextBoolean()) {
+ collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICY, TieredMergePolicy.class.getName());
+ collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICY, "true");
+ collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICYFACTORY, "false");
+ } else {
+ collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY, TieredMergePolicyFactory.class.getName());
+ collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICYFACTORY, "true");
+ collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICY, "false");
+ }
+ collectionProperties.putIfAbsent("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
+ collectionProperties.putIfAbsent("solr.directoryFactory", (persistIndex ? "solr.StandardDirectoryFactory" : "solr.RAMDirectoryFactory"));
+
+ if (asyncId == null) {
+ CollectionAdminRequest.createCollection(collectionName, configName, NUM_SHARDS, REPLICATION_FACTOR)
+ .setCreateNodeSet(createNodeSet)
+ .setProperties(collectionProperties)
+ .process(miniCluster.getSolrClient());
+ }
+ else {
+ CollectionAdminRequest.createCollection(collectionName, configName, NUM_SHARDS, REPLICATION_FACTOR)
+ .setCreateNodeSet(createNodeSet)
+ .setProperties(collectionProperties)
+ .processAndWait(miniCluster.getSolrClient(), 30);
+ }
+ }
+
+
+ public void testSegmentTerminateEarly() throws Exception {
+
+ final String collectionName = "testSegmentTerminateEarlyCollection";
+
+ final SegmentTerminateEarlyTestState tstes = new SegmentTerminateEarlyTestState(random());
+
+ final CloudSolrClient cloudSolrClient = cluster.getSolrClient();
+ cloudSolrClient.setDefaultCollection(collectionName);
+
+ // create collection
+ {
+ final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
+ final Map<String, String> collectionProperties = new HashMap<>();
+ collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml");
+ createCollection(cluster, collectionName, null, asyncId, Boolean.TRUE, collectionProperties);
+ }
+
+ ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
+
+ // add some documents, then optimize to get merged-sorted segments
+ tstes.addDocuments(cloudSolrClient, 10, 10, true);
+
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent
+ tstes.queryTimestampDescending(cloudSolrClient);
+
+ // add a few more documents, but don't optimize to have some not-merge-sorted segments
+ tstes.addDocuments(cloudSolrClient, 2, 10, false);
+
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter now present
+ tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient);
+ tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient);
+
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used
+ tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient);
+ tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order
+
+ }
+}