You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2018/04/16 14:24:15 UTC
[2/2] lucene-solr:master: SOLR-12147: Don't use memory-intensive
codecs to test uninvert limits
SOLR-12147: Don't use memory-intensive codecs to test uninvert limits
This separates out TestDocTermOrds.testTriggerUninvertLimit into its own
test class, and suppresses Memory codec on it
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/37e06500
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/37e06500
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/37e06500
Branch: refs/heads/master
Commit: 37e065007fae77f1e1c13a5c3e2938a7adf7cdf2
Parents: c70ccea
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Apr 16 14:53:35 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Apr 16 15:24:04 2018 +0100
----------------------------------------------------------------------
.../solr/uninverting/TestDocTermOrds.java | 57 -------------
.../TestDocTermOrdsUninvertLimit.java | 88 ++++++++++++++++++++
2 files changed, 88 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37e06500/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
index 21f44ea..0aa91ad 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
-import java.util.Objects;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
@@ -137,62 +136,6 @@ public class TestDocTermOrds extends LuceneTestCase {
dir.close();
}
- /* UnInvertedField had a reference block limitation of 2^24. This unit test triggered it.
- *
- * With the current code, the test verifies that the old limit no longer applies.
- * New limit is 2^31, which is not very realistic to unit-test. */
- @SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"})
- @Nightly
- public void testTriggerUnInvertLimit() throws IOException {
- final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation
-
- assumeFalse("Don't run this massive test with MemoryPostingsFormat, as it can OOM",
- Objects.equals(Codec.getDefault().postingsFormat().getName(), "Memory"));
-
- // Ensure enough terms inside of a single UnInvert-pass-structure to trigger the limit
- final int REF_LIMIT = (int) Math.pow(2, 24); // Maximum number of references within a single pass-structure
- final int DOCS = (1<<16)-1; // The number of documents within a single pass (simplified)
- final int TERMS = REF_LIMIT/DOCS; // Each document must have this many references aka terms hit limit
-
- Directory dir = newDirectory();
- final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
- newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- Document doc = new Document();
- Field field = newTextField("field", "", Field.Store.NO);
- doc.add(field);
-
- StringBuilder sb = new StringBuilder(TERMS*(Integer.toString(TERMS).length()+1));
- for (int i = 0 ; i < TERMS ; i++) {
- sb.append(" ").append(Integer.toString(i));
- }
- field.setStringValue(sb.toString());
-
- for (int i = 0 ; i < DOCS ; i++) {
- w.addDocument(doc);
- }
- //System.out.println("\n Finished adding " + DOCS + " documents of " + TERMS + " unique terms");
- final IndexReader r = w.getReader();
- w.close();
-
- try {
- final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
- TestUtil.checkReader(ar);
- final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); // bigTerms turned off
- if (SHOULD_TRIGGER) {
- fail("DocTermOrds should have failed with a \"Too many values for UnInvertedField\" message");
- }
- } catch (IllegalStateException e) {
- if (!SHOULD_TRIGGER) {
- fail("DocTermsOrd should not have failed with this implementation, but got exception " +
- e.getClass().getSimpleName() + " with message " + e.getMessage());
- }
- // This is (hopefully) "Too many values for UnInvertedField faceting on field field", so all is as expected
- } finally {
- r.close();
- dir.close();
- }
- }
-
public void testRandom() throws Exception {
Directory dir = newDirectory();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37e06500/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
new file mode 100644
index 0000000..af66b42
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.uninverting;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.index.SlowCompositeReaderWrapper;
+
+@LuceneTestCase.SuppressCodecs({"Memory"})
+public class TestDocTermOrdsUninvertLimit extends LuceneTestCase {
+
+ /* UnInvertedField had a reference block limitation of 2^24. This unit test triggered it.
+ *
+ * With the current code, the test verifies that the old limit no longer applies.
+ * New limit is 2^31, which is not very realistic to unit-test. */
+ @SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"})
+ @Nightly
+ public void testTriggerUnInvertLimit() throws IOException {
+ final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation
+
+ // Ensure enough terms inside of a single UnInvert-pass-structure to trigger the limit
+ final int REF_LIMIT = (int) Math.pow(2, 24); // Maximum number of references within a single pass-structure
+ final int DOCS = (1<<16)-1; // The number of documents within a single pass (simplified)
+ final int TERMS = REF_LIMIT/DOCS; // Each document must have this many references aka terms hit limit
+
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
+ newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
+ Document doc = new Document();
+ Field field = newTextField("field", "", Field.Store.NO);
+ doc.add(field);
+
+ StringBuilder sb = new StringBuilder(TERMS*(Integer.toString(TERMS).length()+1));
+ for (int i = 0 ; i < TERMS ; i++) {
+ sb.append(" ").append(Integer.toString(i));
+ }
+ field.setStringValue(sb.toString());
+
+ for (int i = 0 ; i < DOCS ; i++) {
+ w.addDocument(doc);
+ }
+ //System.out.println("\n Finished adding " + DOCS + " documents of " + TERMS + " unique terms");
+ final IndexReader r = w.getReader();
+ w.close();
+
+ try {
+ final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
+ TestUtil.checkReader(ar);
+ final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); // bigTerms turned off
+ if (SHOULD_TRIGGER) {
+ fail("DocTermOrds should have failed with a \"Too many values for UnInvertedField\" message");
+ }
+ } catch (IllegalStateException e) {
+ if (!SHOULD_TRIGGER) {
+ fail("DocTermsOrd should not have failed with this implementation, but got exception " +
+ e.getClass().getSimpleName() + " with message " + e.getMessage());
+ }
+ // This is (hopefully) "Too many values for UnInvertedField faceting on field field", so all is as expected
+ } finally {
+ r.close();
+ dir.close();
+ }
+ }
+}