You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/03/09 17:00:53 UTC
[42/50] [abbrv] lucene-solr git commit: improve testing for sparse
points
improve testing for sparse points
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/62b3aaa5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/62b3aaa5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/62b3aaa5
Branch: refs/heads/apiv2
Commit: 62b3aaa526d6b883e95cf899d0fc89da11fc5e93
Parents: 56ad6e5
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Mar 8 15:21:37 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Mar 8 15:21:37 2016 -0500
----------------------------------------------------------------------
.../index/TestBackwardsCompatibility.java | 2 +-
.../index/TestFlushByRamOrCountsPolicy.java | 2 +-
.../lucene/index/TestForceMergeForever.java | 2 +-
.../test/org/apache/lucene/index/TestNorms.java | 2 +-
.../apache/lucene/index/TestPointValues.java | 46 ++++++++++++++++
.../apache/lucene/index/TestRollingUpdates.java | 2 +-
.../org/apache/lucene/index/TestTermsEnum.java | 2 +-
.../lucene/store/TestNRTCachingDirectory.java | 2 +-
.../org/apache/lucene/util/fst/TestFSTs.java | 2 +-
.../analyzing/TestFreeTextSuggester.java | 2 +-
.../ThreadedIndexingAndSearchingTestCase.java | 2 +-
.../lucene/search/ShardSearchingTestBase.java | 2 +-
.../org/apache/lucene/util/LineFileDocs.java | 58 +++++++++++++-------
13 files changed, 95 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 68b32d3..bc48c7a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -148,7 +148,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
IndexWriterConfig conf = new IndexWriterConfig(analyzer)
.setMergePolicy(mp).setUseCompoundFile(false);
IndexWriter writer = new IndexWriter(dir, conf);
- LineFileDocs docs = new LineFileDocs(null, true);
+ LineFileDocs docs = new LineFileDocs(null);
for(int i=0;i<50;i++) {
writer.addDocument(docs.nextDoc());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
index 562913e..993a521 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
@@ -39,7 +39,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
- lineDocFile = new LineFileDocs(random(), true);
+ lineDocFile = new LineFileDocs(random());
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java b/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java
index 3edeef1..0379395 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java
@@ -62,7 +62,7 @@ public class TestForceMergeForever extends LuceneTestCase {
// Try to make an index that requires merging:
w.getConfig().setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 11));
final int numStartDocs = atLeast(20);
- final LineFileDocs docs = new LineFileDocs(random(), true);
+ final LineFileDocs docs = new LineFileDocs(random());
for(int docIDX=0;docIDX<numStartDocs;docIDX++) {
w.addDocument(docs.nextDoc());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
index 78fc872..562cefb 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
@@ -133,7 +133,7 @@ public class TestNorms extends LuceneTestCase {
Similarity provider = new MySimProvider();
config.setSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
- final LineFileDocs docs = new LineFileDocs(random, true);
+ final LineFileDocs docs = new LineFileDocs(random);
int num = atLeast(100);
for (int i = 0; i < num; i++) {
Document doc = docs.nextDoc();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index 9ced11a..55d4794 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -582,4 +582,50 @@ public class TestPointValues extends LuceneTestCase {
w.close();
dir.close();
}
+
+ public void testSparsePoints() throws Exception {
+ Directory dir = newDirectory();
+ int numDocs = atLeast(1000);
+ int numFields = TestUtil.nextInt(random(), 1, 10);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ int[] fieldDocCounts = new int[numFields];
+ int[] fieldSizes = new int[numFields];
+ for(int i=0;i<numDocs;i++) {
+ Document doc = new Document();
+ for(int field=0;field<numFields;field++) {
+ String fieldName = "int" + field;
+ if (random().nextInt(100) == 17) {
+ doc.add(new IntPoint(fieldName, random().nextInt()));
+ fieldDocCounts[field]++;
+ fieldSizes[field]++;
+
+ if (random().nextInt(10) == 5) {
+ // add same field again!
+ doc.add(new IntPoint(fieldName, random().nextInt()));
+ fieldSizes[field]++;
+ }
+ }
+ }
+ w.addDocument(doc);
+ }
+
+ IndexReader r = w.getReader();
+ for(int field=0;field<numFields;field++) {
+ int docCount = 0;
+ int size = 0;
+ String fieldName = "int" + field;
+ for(LeafReaderContext ctx : r.leaves()) {
+ PointValues points = ctx.reader().getPointValues();
+ if (ctx.reader().getFieldInfos().fieldInfo(fieldName) != null) {
+ docCount += points.getDocCount(fieldName);
+ size += points.size(fieldName);
+ }
+ }
+ assertEquals(fieldDocCounts[field], docCount);
+ assertEquals(fieldSizes[field], size);
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
index e02afc0..23be40b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
@@ -40,7 +40,7 @@ public class TestRollingUpdates extends LuceneTestCase {
Random random = new Random(random().nextLong());
final BaseDirectoryWrapper dir = newDirectory();
- final LineFileDocs docs = new LineFileDocs(random, true);
+ final LineFileDocs docs = new LineFileDocs(random);
//provider.register(new MemoryCodec());
if (random().nextBoolean()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
index 3296330..aa2ca24 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
@@ -41,7 +41,7 @@ public class TestTermsEnum extends LuceneTestCase {
public void test() throws Exception {
Random random = new Random(random().nextLong());
- final LineFileDocs docs = new LineFileDocs(random, true);
+ final LineFileDocs docs = new LineFileDocs(random);
final Directory d = newDirectory();
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
index dfbb7b2..227d41f 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
@@ -56,7 +56,7 @@ public class TestNRTCachingDirectory extends BaseDirectoryTestCase {
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig conf = newIndexWriterConfig(analyzer);
RandomIndexWriter w = new RandomIndexWriter(random(), cachedDir, conf);
- final LineFileDocs docs = new LineFileDocs(random(), true);
+ final LineFileDocs docs = new LineFileDocs(random());
final int numDocs = TestUtil.nextInt(random(), 100, 400);
if (VERBOSE) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
index c352938..5f01482 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -310,7 +310,7 @@ public class TestFSTs extends LuceneTestCase {
// file, up until a doc limit
public void testRealTerms() throws Exception {
- final LineFileDocs docs = new LineFileDocs(random(), true);
+ final LineFileDocs docs = new LineFileDocs(random());
final int numDocs = TEST_NIGHTLY ? atLeast(1000) : atLeast(100);
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
index 4fd7773..b26b5332 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
@@ -131,7 +131,7 @@ public class TestFreeTextSuggester extends LuceneTestCase {
@Ignore
public void testWiki() throws Exception {
- final LineFileDocs lfd = new LineFileDocs(null, "/lucenedata/enwiki/enwiki-20120502-lines-1k.txt", false);
+ final LineFileDocs lfd = new LineFileDocs(null, "/lucenedata/enwiki/enwiki-20120502-lines-1k.txt");
// Skip header:
lfd.nextDoc();
Analyzer analyzer = new MockAnalyzer(random());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
index 80c3903..e79d548 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
@@ -429,7 +429,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
final long t0 = System.currentTimeMillis();
Random random = new Random(random().nextLong());
- final LineFileDocs docs = new LineFileDocs(random, true);
+ final LineFileDocs docs = new LineFileDocs(random);
final Path tempDir = createTempDir(testName);
dir = getDirectory(newMockFSDirectory(tempDir)); // some subclasses rely on this being MDW
if (dir instanceof BaseDirectoryWrapper) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
index 52c16dc..9449a72 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/ShardSearchingTestBase.java
@@ -552,7 +552,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
@Override
public void run() {
try {
- final LineFileDocs docs = new LineFileDocs(random(), true);
+ final LineFileDocs docs = new LineFileDocs(random());
int numDocs = 0;
while (System.nanoTime() < endTimeNanos) {
final int what = random().nextInt(3);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62b3aaa5/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
index 26d7cc3..8844d8f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
@@ -33,16 +33,17 @@ import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
-import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
/** Minimal port of benchmark's LneDocSource +
* DocMaker, so tests can enum docs from a line file created
@@ -53,22 +54,18 @@ public class LineFileDocs implements Closeable {
private final static int BUFFER_SIZE = 1 << 16; // 64K
private final AtomicInteger id = new AtomicInteger();
private final String path;
- private final boolean useDocValues;
+ private final Random random;
/** If forever is true, we rewind the file at EOF (repeat
* the docs over and over) */
- public LineFileDocs(Random random, String path, boolean useDocValues) throws IOException {
+ public LineFileDocs(Random random, String path) throws IOException {
this.path = path;
- this.useDocValues = useDocValues;
+ this.random = new Random(random.nextLong());
open(random);
}
public LineFileDocs(Random random) throws IOException {
- this(random, LuceneTestCase.TEST_LINE_DOCS_FILE, true);
- }
-
- public LineFileDocs(Random random, boolean useDocValues) throws IOException {
- this(random, LuceneTestCase.TEST_LINE_DOCS_FILE, useDocValues);
+ this(random, LuceneTestCase.TEST_LINE_DOCS_FILE);
}
@Override
@@ -165,7 +162,7 @@ public class LineFileDocs implements Closeable {
final Field idNumDV;
final Field date;
- public DocState(boolean useDocValues) {
+ public DocState() {
doc = new Document();
title = new StringField("title", "", Field.Store.NO);
@@ -192,15 +189,10 @@ public class LineFileDocs implements Closeable {
date = new StringField("date", "", Field.Store.YES);
doc.add(date);
- if (useDocValues) {
- titleDV = new SortedDocValuesField("titleDV", new BytesRef());
- idNumDV = new NumericDocValuesField("docid_intDV", 0);
- doc.add(titleDV);
- doc.add(idNumDV);
- } else {
- titleDV = null;
- idNumDV = null;
- }
+ titleDV = new SortedDocValuesField("titleDV", new BytesRef());
+ idNumDV = new NumericDocValuesField("docid_intDV", 0);
+ doc.add(titleDV);
+ doc.add(idNumDV);
}
}
@@ -225,7 +217,7 @@ public class LineFileDocs implements Closeable {
DocState docState = threadDocs.get();
if (docState == null) {
- docState = new DocState(useDocValues);
+ docState = new DocState();
threadDocs.set(docState);
}
@@ -252,6 +244,32 @@ public class LineFileDocs implements Closeable {
if (docState.idNumDV != null) {
docState.idNumDV.setLongValue(i);
}
+
+ if (random.nextInt(5) == 4) {
+ // Make some sparse fields
+ Document doc = new Document();
+ for(IndexableField field : docState.doc) {
+ doc.add(field);
+ }
+
+ if (random.nextInt(3) == 1) {
+ int x = random.nextInt(4);
+ doc.add(new IntPoint("docLength" + x, line.length()));
+ }
+
+ if (random.nextInt(3) == 1) {
+ int x = random.nextInt(4);
+ doc.add(new IntPoint("docTitleLength" + x, title.length()));
+ }
+
+ if (random.nextInt(3) == 1) {
+ int x = random.nextInt(4);
+ doc.add(new NumericDocValuesField("docLength" + x, line.length()));
+ }
+
+ // TODO: more random sparse fields here too
+ }
+
return docState.doc;
}
}