You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/05/25 13:21:16 UTC
[06/10] lucene-solr:master: LUCENE-7283: move
SlowCompositeReaderWrapper and uninverting to solr
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java
deleted file mode 100644
index 9809324..0000000
--- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java
+++ /dev/null
@@ -1,592 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.uninverting;
-
-import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-
-public class TestFieldCacheVsDocValues extends LuceneTestCase {
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- assumeFalse("test unsupported on J9 temporarily, see https://issues.apache.org/jira/browse/LUCENE-6522",
- Constants.JAVA_VENDOR.startsWith("IBM"));
- }
-
- public void testByteMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Byte.MIN_VALUE, Byte.MAX_VALUE);
- }
- }
-
- public void testShortMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Short.MIN_VALUE, Short.MAX_VALUE);
- }
- }
-
- public void testIntMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Integer.MIN_VALUE, Integer.MAX_VALUE);
- }
- }
-
- public void testLongMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Long.MIN_VALUE, Long.MAX_VALUE);
- }
- }
-
- public void testSortedFixedLengthVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- int fixedLength = TestUtil.nextInt(random(), 1, 10);
- doTestSortedVsFieldCache(fixedLength, fixedLength);
- }
- }
-
- public void testSortedVariableLengthVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedVsFieldCache(1, 10);
- }
- }
-
- public void testSortedSetFixedLengthVsUninvertedField() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- int fixedLength = TestUtil.nextInt(random(), 1, 10);
- doTestSortedSetVsUninvertedField(fixedLength, fixedLength);
- }
- }
-
- public void testSortedSetVariableLengthVsUninvertedField() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedSetVsUninvertedField(1, 10);
- }
- }
-
- // LUCENE-4853
- public void testHugeBinaryValues() throws Exception {
- Analyzer analyzer = new MockAnalyzer(random());
- // FSDirectory because SimpleText will consume gobbs of
- // space when storing big binary values:
- Directory d = newFSDirectory(createTempDir("hugeBinaryValues"));
- boolean doFixed = random().nextBoolean();
- int numDocs;
- int fixedLength = 0;
- if (doFixed) {
- // Sometimes make all values fixed length since some
- // codecs have different code paths for this:
- numDocs = TestUtil.nextInt(random(), 10, 20);
- fixedLength = TestUtil.nextInt(random(), 65537, 256 * 1024);
- } else {
- numDocs = TestUtil.nextInt(random(), 100, 200);
- }
- IndexWriter w = new IndexWriter(d, newIndexWriterConfig(analyzer));
- List<byte[]> docBytes = new ArrayList<>();
- long totalBytes = 0;
- for(int docID=0;docID<numDocs;docID++) {
- // we don't use RandomIndexWriter because it might add
- // more docvalues than we expect !!!!
-
- // Must be > 64KB in size to ensure more than 2 pages in
- // PagedBytes would be needed:
- int numBytes;
- if (doFixed) {
- numBytes = fixedLength;
- } else if (docID == 0 || random().nextInt(5) == 3) {
- numBytes = TestUtil.nextInt(random(), 65537, 3 * 1024 * 1024);
- } else {
- numBytes = TestUtil.nextInt(random(), 1, 1024 * 1024);
- }
- totalBytes += numBytes;
- if (totalBytes > 5 * 1024*1024) {
- break;
- }
- byte[] bytes = new byte[numBytes];
- random().nextBytes(bytes);
- docBytes.add(bytes);
- Document doc = new Document();
- BytesRef b = new BytesRef(bytes);
- b.length = bytes.length;
- doc.add(new BinaryDocValuesField("field", b));
- doc.add(new StringField("id", ""+docID, Field.Store.YES));
- try {
- w.addDocument(doc);
- } catch (IllegalArgumentException iae) {
- if (iae.getMessage().indexOf("is too large") == -1) {
- throw iae;
- } else {
- // OK: some codecs can't handle binary DV > 32K
- assertFalse(codecAcceptsHugeBinaryValues("field"));
- w.rollback();
- d.close();
- return;
- }
- }
- }
-
- DirectoryReader r;
- try {
- r = DirectoryReader.open(w);
- } catch (IllegalArgumentException iae) {
- if (iae.getMessage().indexOf("is too large") == -1) {
- throw iae;
- } else {
- assertFalse(codecAcceptsHugeBinaryValues("field"));
-
- // OK: some codecs can't handle binary DV > 32K
- w.rollback();
- d.close();
- return;
- }
- }
- w.close();
-
- LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
- TestUtil.checkReader(ar);
-
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
- for(int docID=0;docID<docBytes.size();docID++) {
- Document doc = ar.document(docID);
- BytesRef bytes = s.get(docID);
- byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
- assertEquals(expected.length, bytes.length);
- assertEquals(new BytesRef(expected), bytes);
- }
-
- assertTrue(codecAcceptsHugeBinaryValues("field"));
-
- ar.close();
- d.close();
- }
-
- private static final int LARGE_BINARY_FIELD_LENGTH = (1 << 15) - 2;
-
- // TODO: get this out of here and into the deprecated codecs (4.0, 4.2)
- public void testHugeBinaryValueLimit() throws Exception {
- // We only test DVFormats that have a limit
- assumeFalse("test requires codec with limits on max binary field length", codecAcceptsHugeBinaryValues("field"));
- Analyzer analyzer = new MockAnalyzer(random());
- // FSDirectory because SimpleText will consume gobbs of
- // space when storing big binary values:
- Directory d = newFSDirectory(createTempDir("hugeBinaryValues"));
- boolean doFixed = random().nextBoolean();
- int numDocs;
- int fixedLength = 0;
- if (doFixed) {
- // Sometimes make all values fixed length since some
- // codecs have different code paths for this:
- numDocs = TestUtil.nextInt(random(), 10, 20);
- fixedLength = LARGE_BINARY_FIELD_LENGTH;
- } else {
- numDocs = TestUtil.nextInt(random(), 100, 200);
- }
- IndexWriter w = new IndexWriter(d, newIndexWriterConfig(analyzer));
- List<byte[]> docBytes = new ArrayList<>();
- long totalBytes = 0;
- for(int docID=0;docID<numDocs;docID++) {
- // we don't use RandomIndexWriter because it might add
- // more docvalues than we expect !!!!
-
- // Must be > 64KB in size to ensure more than 2 pages in
- // PagedBytes would be needed:
- int numBytes;
- if (doFixed) {
- numBytes = fixedLength;
- } else if (docID == 0 || random().nextInt(5) == 3) {
- numBytes = LARGE_BINARY_FIELD_LENGTH;
- } else {
- numBytes = TestUtil.nextInt(random(), 1, LARGE_BINARY_FIELD_LENGTH);
- }
- totalBytes += numBytes;
- if (totalBytes > 5 * 1024*1024) {
- break;
- }
- byte[] bytes = new byte[numBytes];
- random().nextBytes(bytes);
- docBytes.add(bytes);
- Document doc = new Document();
- BytesRef b = new BytesRef(bytes);
- b.length = bytes.length;
- doc.add(new BinaryDocValuesField("field", b));
- doc.add(new StringField("id", ""+docID, Field.Store.YES));
- w.addDocument(doc);
- }
-
- DirectoryReader r = DirectoryReader.open(w);
- w.close();
-
- LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
- TestUtil.checkReader(ar
- );
-
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
- for(int docID=0;docID<docBytes.size();docID++) {
- Document doc = ar.document(docID);
- BytesRef bytes = s.get(docID);
- byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
- assertEquals(expected.length, bytes.length);
- assertEquals(new BytesRef(expected), bytes);
- }
-
- ar.close();
- d.close();
- }
-
- private void doTestSortedVsFieldCache(int minLength, int maxLength) throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
- Document doc = new Document();
- Field idField = new StringField("id", "", Field.Store.NO);
- Field indexedField = new StringField("indexed", "", Field.Store.NO);
- Field dvField = new SortedDocValuesField("dv", new BytesRef());
- doc.add(idField);
- doc.add(indexedField);
- doc.add(dvField);
-
- // index some docs
- int numDocs = atLeast(300);
- for (int i = 0; i < numDocs; i++) {
- idField.setStringValue(Integer.toString(i));
- final int length;
- if (minLength == maxLength) {
- length = minLength; // fixed length
- } else {
- length = TestUtil.nextInt(random(), minLength, maxLength);
- }
- String value = TestUtil.randomSimpleString(random(), length);
- indexedField.setStringValue(value);
- dvField.setBytesValue(new BytesRef(value));
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs/10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
- writer.close();
-
- // compare
- DirectoryReader ir = DirectoryReader.open(dir);
- for (LeafReaderContext context : ir.leaves()) {
- LeafReader r = context.reader();
- SortedDocValues expected = FieldCache.DEFAULT.getTermsIndex(r, "indexed");
- SortedDocValues actual = r.getSortedDocValues("dv");
- assertEquals(r.maxDoc(), expected, actual);
- }
- ir.close();
- dir.close();
- }
-
- private void doTestSortedSetVsUninvertedField(int minLength, int maxLength) throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
-
- // index some docs
- int numDocs = atLeast(300);
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- Field idField = new StringField("id", Integer.toString(i), Field.Store.NO);
- doc.add(idField);
- final int length = TestUtil.nextInt(random(), minLength, maxLength);
- int numValues = random().nextInt(17);
- // create a random list of strings
- List<String> values = new ArrayList<>();
- for (int v = 0; v < numValues; v++) {
- values.add(TestUtil.randomSimpleString(random(), minLength, length));
- }
-
- // add in any order to the indexed field
- ArrayList<String> unordered = new ArrayList<>(values);
- Collections.shuffle(unordered, random());
- for (String v : values) {
- doc.add(newStringField("indexed", v, Field.Store.NO));
- }
-
- // add in any order to the dv field
- ArrayList<String> unordered2 = new ArrayList<>(values);
- Collections.shuffle(unordered2, random());
- for (String v : unordered2) {
- doc.add(new SortedSetDocValuesField("dv", new BytesRef(v)));
- }
-
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs/10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
-
- // compare per-segment
- DirectoryReader ir = writer.getReader();
- for (LeafReaderContext context : ir.leaves()) {
- LeafReader r = context.reader();
- SortedSetDocValues expected = FieldCache.DEFAULT.getDocTermOrds(r, "indexed", null);
- SortedSetDocValues actual = r.getSortedSetDocValues("dv");
- assertEquals(r.maxDoc(), expected, actual);
- }
- ir.close();
-
- writer.forceMerge(1);
-
- // now compare again after the merge
- ir = writer.getReader();
- LeafReader ar = getOnlyLeafReader(ir);
- SortedSetDocValues expected = FieldCache.DEFAULT.getDocTermOrds(ar, "indexed", null);
- SortedSetDocValues actual = ar.getSortedSetDocValues("dv");
- assertEquals(ir.maxDoc(), expected, actual);
- ir.close();
-
- writer.close();
- dir.close();
- }
-
- private void doTestMissingVsFieldCache(LongProducer longs) throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
- Field idField = new StringField("id", "", Field.Store.NO);
- Field indexedField = newStringField("indexed", "", Field.Store.NO);
- Field dvField = new NumericDocValuesField("dv", 0);
-
-
- // index some docs
- int numDocs = atLeast(300);
- // numDocs should be always > 256 so that in case of a codec that optimizes
- // for numbers of values <= 256, all storage layouts are tested
- assert numDocs > 256;
- for (int i = 0; i < numDocs; i++) {
- idField.setStringValue(Integer.toString(i));
- long value = longs.next();
- indexedField.setStringValue(Long.toString(value));
- dvField.setLongValue(value);
- Document doc = new Document();
- doc.add(idField);
- // 1/4 of the time we neglect to add the fields
- if (random().nextInt(4) > 0) {
- doc.add(indexedField);
- doc.add(dvField);
- }
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs/10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
-
- // merge some segments and ensure that at least one of them has more than
- // 256 values
- writer.forceMerge(numDocs / 256);
-
- writer.close();
-
- // compare
- DirectoryReader ir = DirectoryReader.open(dir);
- for (LeafReaderContext context : ir.leaves()) {
- LeafReader r = context.reader();
- Bits expected = FieldCache.DEFAULT.getDocsWithField(r, "indexed", null);
- Bits actual = FieldCache.DEFAULT.getDocsWithField(r, "dv", null);
- assertEquals(expected, actual);
- }
- ir.close();
- dir.close();
- }
-
- private void doTestMissingVsFieldCache(final long minValue, final long maxValue) throws Exception {
- doTestMissingVsFieldCache(new LongProducer() {
- @Override
- long next() {
- return TestUtil.nextLong(random(), minValue, maxValue);
- }
- });
- }
-
- static abstract class LongProducer {
- abstract long next();
- }
-
- private void assertEquals(Bits expected, Bits actual) throws Exception {
- assertEquals(expected.length(), actual.length());
- for (int i = 0; i < expected.length(); i++) {
- assertEquals(expected.get(i), actual.get(i));
- }
- }
-
- private void assertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual) throws Exception {
- assertEquals(maxDoc, DocValues.singleton(expected), DocValues.singleton(actual));
- }
-
- private void assertEquals(int maxDoc, SortedSetDocValues expected, SortedSetDocValues actual) throws Exception {
- // can be null for the segment if no docs actually had any SortedDocValues
- // in this case FC.getDocTermsOrds returns EMPTY
- if (actual == null) {
- assertEquals(expected.getValueCount(), 0);
- return;
- }
- assertEquals(expected.getValueCount(), actual.getValueCount());
- // compare ord lists
- for (int i = 0; i < maxDoc; i++) {
- expected.setDocument(i);
- actual.setDocument(i);
- long expectedOrd;
- while ((expectedOrd = expected.nextOrd()) != NO_MORE_ORDS) {
- assertEquals(expectedOrd, actual.nextOrd());
- }
- assertEquals(NO_MORE_ORDS, actual.nextOrd());
- }
-
- // compare ord dictionary
- for (long i = 0; i < expected.getValueCount(); i++) {
- final BytesRef expectedBytes = BytesRef.deepCopyOf(expected.lookupOrd(i));
- final BytesRef actualBytes = actual.lookupOrd(i);
- assertEquals(expectedBytes, actualBytes);
- }
-
- // compare termsenum
- assertEquals(expected.getValueCount(), expected.termsEnum(), actual.termsEnum());
- }
-
- private void assertEquals(long numOrds, TermsEnum expected, TermsEnum actual) throws Exception {
- BytesRef ref;
-
- // sequential next() through all terms
- while ((ref = expected.next()) != null) {
- assertEquals(ref, actual.next());
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
- assertNull(actual.next());
-
- // sequential seekExact(ord) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- actual.seekExact(i);
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // sequential seekExact(BytesRef) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- assertTrue(actual.seekExact(expected.term()));
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // sequential seekCeil(BytesRef) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- assertEquals(SeekStatus.FOUND, actual.seekCeil(expected.term()));
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekExact(ord)
- for (long i = 0; i < numOrds; i++) {
- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
- expected.seekExact(randomOrd);
- actual.seekExact(randomOrd);
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekExact(BytesRef)
- for (long i = 0; i < numOrds; i++) {
- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
- expected.seekExact(randomOrd);
- actual.seekExact(expected.term());
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekCeil(BytesRef)
- for (long i = 0; i < numOrds; i++) {
- BytesRef target = new BytesRef(TestUtil.randomUnicodeString(random()));
- SeekStatus expectedStatus = expected.seekCeil(target);
- assertEquals(expectedStatus, actual.seekCeil(target));
- if (expectedStatus != SeekStatus.END) {
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
- }
- }
-
- protected boolean codecAcceptsHugeBinaryValues(String field) {
- String name = TestUtil.getDocValuesFormat(field);
- return !(name.equals("Memory")); // Direct has a different type of limit
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java
deleted file mode 100644
index 3f1f450..0000000
--- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.uninverting;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-
-// TODO: what happened to this test... its not actually uninverting?
-public class TestFieldCacheWithThreads extends LuceneTestCase {
-
- public void test() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
-
- final List<Long> numbers = new ArrayList<>();
- final List<BytesRef> binary = new ArrayList<>();
- final List<BytesRef> sorted = new ArrayList<>();
- final int numDocs = atLeast(100);
- for(int i=0;i<numDocs;i++) {
- Document d = new Document();
- long number = random().nextLong();
- d.add(new NumericDocValuesField("number", number));
- BytesRef bytes = new BytesRef(TestUtil.randomRealisticUnicodeString(random()));
- d.add(new BinaryDocValuesField("bytes", bytes));
- binary.add(bytes);
- bytes = new BytesRef(TestUtil.randomRealisticUnicodeString(random()));
- d.add(new SortedDocValuesField("sorted", bytes));
- sorted.add(bytes);
- w.addDocument(d);
- numbers.add(number);
- }
-
- w.forceMerge(1);
- final IndexReader r = DirectoryReader.open(w);
- w.close();
-
- assertEquals(1, r.leaves().size());
- final LeafReader ar = r.leaves().get(0).reader();
-
- int numThreads = TestUtil.nextInt(random(), 2, 5);
- List<Thread> threads = new ArrayList<>();
- final CountDownLatch startingGun = new CountDownLatch(1);
- for(int t=0;t<numThreads;t++) {
- final Random threadRandom = new Random(random().nextLong());
- Thread thread = new Thread() {
- @Override
- public void run() {
- try {
- //NumericDocValues ndv = ar.getNumericDocValues("number");
- NumericDocValues ndv = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER, false);
- //BinaryDocValues bdv = ar.getBinaryDocValues("bytes");
- BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes", false);
- SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
- startingGun.await();
- int iters = atLeast(1000);
- for(int iter=0;iter<iters;iter++) {
- int docID = threadRandom.nextInt(numDocs);
- switch(threadRandom.nextInt(4)) {
- case 0:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.INT_POINT_PARSER, false).get(docID));
- break;
- case 1:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER, false).get(docID));
- break;
- case 2:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.FLOAT_POINT_PARSER, false).get(docID));
- break;
- case 3:
- assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.DOUBLE_POINT_PARSER, false).get(docID));
- break;
- }
- BytesRef term = bdv.get(docID);
- assertEquals(binary.get(docID), term);
- term = sdv.get(docID);
- assertEquals(sorted.get(docID), term);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- };
- thread.start();
- threads.add(thread);
- }
-
- startingGun.countDown();
-
- for(Thread thread : threads) {
- thread.join();
- }
-
- r.close();
- dir.close();
- }
-
- public void test2() throws Exception {
- Random random = random();
- final int NUM_DOCS = atLeast(100);
- final Directory dir = newDirectory();
- final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
- final boolean allowDups = random.nextBoolean();
- final Set<String> seen = new HashSet<>();
- if (VERBOSE) {
- System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " allowDups=" + allowDups);
- }
- int numDocs = 0;
- final List<BytesRef> docValues = new ArrayList<>();
-
- // TODO: deletions
- while (numDocs < NUM_DOCS) {
- final String s;
- if (random.nextBoolean()) {
- s = TestUtil.randomSimpleString(random);
- } else {
- s = TestUtil.randomUnicodeString(random);
- }
- final BytesRef br = new BytesRef(s);
-
- if (!allowDups) {
- if (seen.contains(s)) {
- continue;
- }
- seen.add(s);
- }
-
- if (VERBOSE) {
- System.out.println(" " + numDocs + ": s=" + s);
- }
-
- final Document doc = new Document();
- doc.add(new SortedDocValuesField("stringdv", br));
- doc.add(new NumericDocValuesField("id", numDocs));
- docValues.add(br);
- writer.addDocument(doc);
- numDocs++;
-
- if (random.nextInt(40) == 17) {
- // force flush
- writer.getReader().close();
- }
- }
-
- writer.forceMerge(1);
- final DirectoryReader r = writer.getReader();
- writer.close();
-
- final LeafReader sr = getOnlyLeafReader(r);
-
- final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
-
- final int NUM_THREADS = TestUtil.nextInt(random(), 1, 10);
- Thread[] threads = new Thread[NUM_THREADS];
- for(int thread=0;thread<NUM_THREADS;thread++) {
- threads[thread] = new Thread() {
- @Override
- public void run() {
- Random random = random();
- final SortedDocValues stringDVDirect;
- final NumericDocValues docIDToID;
- try {
- stringDVDirect = sr.getSortedDocValues("stringdv");
- docIDToID = sr.getNumericDocValues("id");
- assertNotNull(stringDVDirect);
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- while(System.currentTimeMillis() < END_TIME) {
- final SortedDocValues source;
- source = stringDVDirect;
-
- for(int iter=0;iter<100;iter++) {
- final int docID = random.nextInt(sr.maxDoc());
- BytesRef term = source.get(docID);
- assertEquals(docValues.get((int) docIDToID.get(docID)), term);
- }
- }
- }
- };
- threads[thread].start();
- }
-
- for(Thread thread : threads) {
- thread.join();
- }
-
- r.close();
- dir.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java
deleted file mode 100644
index 4fd66ad..0000000
--- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.uninverting;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.LegacyDoubleField;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.LegacyFloatField;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.document.LegacyLongField;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LegacyNumericUtils;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-/** random assortment of tests against legacy numerics */
-public class TestLegacyFieldCache extends LuceneTestCase {
- private static LeafReader reader;
- private static int NUM_DOCS;
- private static Directory directory;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- NUM_DOCS = atLeast(500);
- directory = newDirectory();
- RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- long theLong = Long.MAX_VALUE;
- double theDouble = Double.MAX_VALUE;
- int theInt = Integer.MAX_VALUE;
- float theFloat = Float.MAX_VALUE;
- if (VERBOSE) {
- System.out.println("TEST: setUp");
- }
- for (int i = 0; i < NUM_DOCS; i++){
- Document doc = new Document();
- doc.add(new LegacyLongField("theLong", theLong--, Field.Store.NO));
- doc.add(new LegacyDoubleField("theDouble", theDouble--, Field.Store.NO));
- doc.add(new LegacyIntField("theInt", theInt--, Field.Store.NO));
- doc.add(new LegacyFloatField("theFloat", theFloat--, Field.Store.NO));
- if (i%2 == 0) {
- doc.add(new LegacyIntField("sparse", i, Field.Store.NO));
- }
-
- if (i%2 == 0) {
- doc.add(new LegacyIntField("numInt", i, Field.Store.NO));
- }
- writer.addDocument(doc);
- }
- IndexReader r = writer.getReader();
- reader = SlowCompositeReaderWrapper.wrap(r);
- TestUtil.checkReader(reader);
- writer.close();
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- reader.close();
- reader = null;
- directory.close();
- directory = null;
- }
-
- public void testInfoStream() throws Exception {
- try {
- FieldCache cache = FieldCache.DEFAULT;
- ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
- cache.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
- cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false);
- cache.getNumerics(reader, "theDouble", new FieldCache.Parser() {
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return LegacyNumericUtils.filterPrefixCodedLongs(terms.iterator());
- }
- @Override
- public long parseValue(BytesRef term) {
- int val = (int) LegacyNumericUtils.prefixCodedToLong(term);
- if (val<0) val ^= 0x7fffffff;
- return val;
- }
- }, false);
- assertTrue(bos.toString(IOUtils.UTF_8).indexOf("WARNING") != -1);
- } finally {
- FieldCache.DEFAULT.setInfoStream(null);
- FieldCache.DEFAULT.purgeAllCaches();
- }
- }
-
- public void test() throws IOException {
- FieldCache cache = FieldCache.DEFAULT;
- NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", doubles, cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, random().nextBoolean()));
- for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.get(i));
- }
-
- NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", longs, cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER, random().nextBoolean()));
- for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Long.MAX_VALUE - i, longs.get(i));
- }
-
- NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", ints, cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean()));
- for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Integer.MAX_VALUE - i, ints.get(i));
- }
-
- NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER, random().nextBoolean());
- assertSame("Second request to cache return same array", floats, cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER, random().nextBoolean()));
- for (int i = 0; i < NUM_DOCS; i++) {
- assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.get(i));
- }
-
- Bits docsWithField = cache.getDocsWithField(reader, "theLong", null);
- assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "theLong", null));
- assertTrue("docsWithField(theLong) must be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits);
- assertTrue("docsWithField(theLong) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS);
- for (int i = 0; i < docsWithField.length(); i++) {
- assertTrue(docsWithField.get(i));
- }
-
- docsWithField = cache.getDocsWithField(reader, "sparse", null);
- assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "sparse", null));
- assertFalse("docsWithField(sparse) must not be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits);
- assertTrue("docsWithField(sparse) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS);
- for (int i = 0; i < docsWithField.length(); i++) {
- assertEquals(i%2 == 0, docsWithField.get(i));
- }
-
- FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
- }
-
- public void testEmptyIndex() throws Exception {
- Directory dir = newDirectory();
- IndexWriter writer= new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(500));
- writer.close();
- IndexReader r = DirectoryReader.open(dir);
- LeafReader reader = SlowCompositeReaderWrapper.wrap(r);
- TestUtil.checkReader(reader);
- FieldCache.DEFAULT.getTerms(reader, "foobar", true);
- FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
- FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
- r.close();
- dir.close();
- }
-
- public void testDocsWithField() throws Exception {
- FieldCache cache = FieldCache.DEFAULT;
- cache.purgeAllCaches();
- assertEquals(0, cache.getCacheEntries().length);
- cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, true);
-
- // The double[] takes one slots, and docsWithField should also
- // have been populated:
- assertEquals(2, cache.getCacheEntries().length);
- Bits bits = cache.getDocsWithField(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER);
-
- // No new entries should appear:
- assertEquals(2, cache.getCacheEntries().length);
- assertTrue(bits instanceof Bits.MatchAllBits);
-
- NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER, true);
- assertEquals(4, cache.getCacheEntries().length);
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.LEGACY_INT_PARSER);
- assertEquals(4, cache.getCacheEntries().length);
- for (int i = 0; i < docsWithField.length(); i++) {
- if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, ints.get(i));
- } else {
- assertFalse(docsWithField.get(i));
- }
- }
-
- NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean());
- docsWithField = cache.getDocsWithField(reader, "numInt", FieldCache.LEGACY_INT_PARSER);
- for (int i = 0; i < docsWithField.length(); i++) {
- if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, numInts.get(i));
- } else {
- assertFalse(docsWithField.get(i));
- }
- }
- }
-
- public void testGetDocsWithFieldThreadSafety() throws Exception {
- final FieldCache cache = FieldCache.DEFAULT;
- cache.purgeAllCaches();
-
- int NUM_THREADS = 3;
- Thread[] threads = new Thread[NUM_THREADS];
- final AtomicBoolean failed = new AtomicBoolean();
- final AtomicInteger iters = new AtomicInteger();
- final int NUM_ITER = 200 * RANDOM_MULTIPLIER;
- final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS,
- new Runnable() {
- @Override
- public void run() {
- cache.purgeAllCaches();
- iters.incrementAndGet();
- }
- });
- for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) {
- threads[threadIDX] = new Thread() {
- @Override
- public void run() {
-
- try {
- while(!failed.get()) {
- final int op = random().nextInt(3);
- if (op == 0) {
- // Purge all caches & resume, once all
- // threads get here:
- restart.await();
- if (iters.get() >= NUM_ITER) {
- break;
- }
- } else if (op == 1) {
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", null);
- for (int i = 0; i < docsWithField.length(); i++) {
- assertEquals(i%2 == 0, docsWithField.get(i));
- }
- } else {
- NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER, true);
- Bits docsWithField = cache.getDocsWithField(reader, "sparse", null);
- for (int i = 0; i < docsWithField.length(); i++) {
- if (i%2 == 0) {
- assertTrue(docsWithField.get(i));
- assertEquals(i, ints.get(i));
- } else {
- assertFalse(docsWithField.get(i));
- }
- }
- }
- }
- } catch (Throwable t) {
- failed.set(true);
- restart.reset();
- throw new RuntimeException(t);
- }
- }
- };
- threads[threadIDX].start();
- }
-
- for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) {
- threads[threadIDX].join();
- }
- assertFalse(failed.get());
- }
-
- public void testDocValuesIntegration() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(null);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
- Document doc = new Document();
- doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value")));
- doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value")));
- doc.add(new NumericDocValuesField("numeric", 42));
- doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
- doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
- iw.addDocument(doc);
- DirectoryReader ir = iw.getReader();
- iw.close();
- LeafReader ar = getOnlyLeafReader(ir);
-
- // Binary type: can be retrieved via getTerms()
- expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.LEGACY_INT_PARSER, false);
- });
-
- // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
- expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.LEGACY_INT_PARSER, false);
- });
-
- // Numeric type: can be retrieved via getInts() and so on
- NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.LEGACY_INT_PARSER, false);
- assertEquals(42, numeric.get(0));
-
- // SortedSet type: can be retrieved via getDocTermOrds()
- expectThrows(IllegalStateException.class, () -> {
- FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.LEGACY_INT_PARSER, false);
- });
-
- ir.close();
- dir.close();
- }
-
- public void testNonexistantFields() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
- iw.addDocument(doc);
- DirectoryReader ir = iw.getReader();
- iw.close();
-
- LeafReader ar = getOnlyLeafReader(ir);
-
- final FieldCache cache = FieldCache.DEFAULT;
- cache.purgeAllCaches();
- assertEquals(0, cache.getCacheEntries().length);
-
- NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER, true);
- assertEquals(0, ints.get(0));
-
- NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER, true);
- assertEquals(0, longs.get(0));
-
- NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER, true);
- assertEquals(0, floats.get(0));
-
- NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER, true);
- assertEquals(0, doubles.get(0));
-
- // check that we cached nothing
- assertEquals(0, cache.getCacheEntries().length);
- ir.close();
- dir.close();
- }
-
- public void testNonIndexedFields() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
- doc.add(new StoredField("bogusbytes", "bogus"));
- doc.add(new StoredField("bogusshorts", "bogus"));
- doc.add(new StoredField("bogusints", "bogus"));
- doc.add(new StoredField("boguslongs", "bogus"));
- doc.add(new StoredField("bogusfloats", "bogus"));
- doc.add(new StoredField("bogusdoubles", "bogus"));
- doc.add(new StoredField("bogusbits", "bogus"));
- iw.addDocument(doc);
- DirectoryReader ir = iw.getReader();
- iw.close();
-
- LeafReader ar = getOnlyLeafReader(ir);
-
- final FieldCache cache = FieldCache.DEFAULT;
- cache.purgeAllCaches();
- assertEquals(0, cache.getCacheEntries().length);
-
- NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER, true);
- assertEquals(0, ints.get(0));
-
- NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER, true);
- assertEquals(0, longs.get(0));
-
- NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER, true);
- assertEquals(0, floats.get(0));
-
- NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER, true);
- assertEquals(0, doubles.get(0));
-
- // check that we cached nothing
- assertEquals(0, cache.getCacheEntries().length);
- ir.close();
- dir.close();
- }
-
- // Make sure that the use of GrowableWriter doesn't prevent from using the full long range
- public void testLongFieldCache() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
- cfg.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg);
- Document doc = new Document();
- LegacyLongField field = new LegacyLongField("f", 0L, Store.YES);
- doc.add(field);
- final long[] values = new long[TestUtil.nextInt(random(), 1, 10)];
- for (int i = 0; i < values.length; ++i) {
- final long v;
- switch (random().nextInt(10)) {
- case 0:
- v = Long.MIN_VALUE;
- break;
- case 1:
- v = 0;
- break;
- case 2:
- v = Long.MAX_VALUE;
- break;
- default:
- v = TestUtil.nextLong(random(), -10, 10);
- break;
- }
- values[i] = v;
- if (v == 0 && random().nextBoolean()) {
- // missing
- iw.addDocument(new Document());
- } else {
- field.setLongValue(v);
- iw.addDocument(doc);
- }
- }
- iw.forceMerge(1);
- final DirectoryReader reader = iw.getReader();
- final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_LONG_PARSER, false);
- for (int i = 0; i < values.length; ++i) {
- assertEquals(values[i], longs.get(i));
- }
- reader.close();
- iw.close();
- dir.close();
- }
-
- // Make sure that the use of GrowableWriter doesn't prevent from using the full int range
- public void testIntFieldCache() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
- cfg.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg);
- Document doc = new Document();
- LegacyIntField field = new LegacyIntField("f", 0, Store.YES);
- doc.add(field);
- final int[] values = new int[TestUtil.nextInt(random(), 1, 10)];
- for (int i = 0; i < values.length; ++i) {
- final int v;
- switch (random().nextInt(10)) {
- case 0:
- v = Integer.MIN_VALUE;
- break;
- case 1:
- v = 0;
- break;
- case 2:
- v = Integer.MAX_VALUE;
- break;
- default:
- v = TestUtil.nextInt(random(), -10, 10);
- break;
- }
- values[i] = v;
- if (v == 0 && random().nextBoolean()) {
- // missing
- iw.addDocument(new Document());
- } else {
- field.setIntValue(v);
- iw.addDocument(doc);
- }
- }
- iw.forceMerge(1);
- final DirectoryReader reader = iw.getReader();
- final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_INT_PARSER, false);
- for (int i = 0; i < values.length; ++i) {
- assertEquals(values[i], ints.get(i));
- }
- reader.close();
- iw.close();
- dir.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java
deleted file mode 100644
index a0cddf8..0000000
--- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.uninverting;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LegacyNumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.uninverting.UninvertingReader.Type;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestNumericTerms32 extends LuceneTestCase {
- // distance of entries
- private static int distance;
- // shift the starting of the values to the left, to also have negative values:
- private static final int startOffset = - 1 << 15;
- // number of docs to generate for testing
- private static int noDocs;
-
- private static Directory directory = null;
- private static IndexReader reader = null;
- private static IndexSearcher searcher = null;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- noDocs = atLeast(4096);
- distance = (1 << 30) / noDocs;
- directory = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000))
- .setMergePolicy(newLogMergePolicy()));
-
- final FieldType storedInt = new FieldType(LegacyIntField.TYPE_NOT_STORED);
- storedInt.setStored(true);
- storedInt.freeze();
-
- final FieldType storedInt8 = new FieldType(storedInt);
- storedInt8.setNumericPrecisionStep(8);
-
- final FieldType storedInt4 = new FieldType(storedInt);
- storedInt4.setNumericPrecisionStep(4);
-
- final FieldType storedInt2 = new FieldType(storedInt);
- storedInt2.setNumericPrecisionStep(2);
-
- LegacyIntField
- field8 = new LegacyIntField("field8", 0, storedInt8),
- field4 = new LegacyIntField("field4", 0, storedInt4),
- field2 = new LegacyIntField("field2", 0, storedInt2);
-
- Document doc = new Document();
- // add fields, that have a distance to test general functionality
- doc.add(field8); doc.add(field4); doc.add(field2);
-
- // Add a series of noDocs docs with increasing int values
- for (int l=0; l<noDocs; l++) {
- int val=distance*l+startOffset;
- field8.setIntValue(val);
- field4.setIntValue(val);
- field2.setIntValue(val);
-
- val=l-(noDocs/2);
- writer.addDocument(doc);
- }
-
- Map<String,Type> map = new HashMap<>();
- map.put("field2", Type.LEGACY_INTEGER);
- map.put("field4", Type.LEGACY_INTEGER);
- map.put("field8", Type.LEGACY_INTEGER);
- reader = UninvertingReader.wrap(writer.getReader(), map);
- searcher=newSearcher(reader);
- writer.close();
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- searcher = null;
- TestUtil.checkReader(reader);
- reader.close();
- reader = null;
- directory.close();
- directory = null;
- }
-
- private void testSorting(int precisionStep) throws Exception {
- String field="field"+precisionStep;
- // 10 random tests, the index order is ascending,
- // so using a reverse sort field should retun descending documents
- int num = TestUtil.nextInt(random(), 10, 20);
- for (int i = 0; i < num; i++) {
- int lower=(int)(random().nextDouble()*noDocs*distance)+startOffset;
- int upper=(int)(random().nextDouble()*noDocs*distance)+startOffset;
- if (lower>upper) {
- int a=lower; lower=upper; upper=a;
- }
- Query tq= LegacyNumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, true);
- TopDocs topDocs = searcher.search(tq, noDocs, new Sort(new SortField(field, SortField.Type.INT, true)));
- if (topDocs.totalHits==0) continue;
- ScoreDoc[] sd = topDocs.scoreDocs;
- assertNotNull(sd);
- int last = searcher.doc(sd[0].doc).getField(field).numericValue().intValue();
- for (int j=1; j<sd.length; j++) {
- int act = searcher.doc(sd[j].doc).getField(field).numericValue().intValue();
- assertTrue("Docs should be sorted backwards", last>act );
- last=act;
- }
- }
- }
-
- @Test
- public void testSorting_8bit() throws Exception {
- testSorting(8);
- }
-
- @Test
- public void testSorting_4bit() throws Exception {
- testSorting(4);
- }
-
- @Test
- public void testSorting_2bit() throws Exception {
- testSorting(2);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java
deleted file mode 100644
index 0724d86..0000000
--- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.uninverting;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.LegacyLongField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LegacyNumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.uninverting.UninvertingReader.Type;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestNumericTerms64 extends LuceneTestCase {
- // distance of entries
- private static long distance;
- // shift the starting of the values to the left, to also have negative values:
- private static final long startOffset = - 1L << 31;
- // number of docs to generate for testing
- private static int noDocs;
-
- private static Directory directory = null;
- private static IndexReader reader = null;
- private static IndexSearcher searcher = null;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- noDocs = atLeast(4096);
- distance = (1L << 60) / noDocs;
- directory = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000))
- .setMergePolicy(newLogMergePolicy()));
-
- final FieldType storedLong = new FieldType(LegacyLongField.TYPE_NOT_STORED);
- storedLong.setStored(true);
- storedLong.freeze();
-
- final FieldType storedLong8 = new FieldType(storedLong);
- storedLong8.setNumericPrecisionStep(8);
-
- final FieldType storedLong4 = new FieldType(storedLong);
- storedLong4.setNumericPrecisionStep(4);
-
- final FieldType storedLong6 = new FieldType(storedLong);
- storedLong6.setNumericPrecisionStep(6);
-
- final FieldType storedLong2 = new FieldType(storedLong);
- storedLong2.setNumericPrecisionStep(2);
-
- LegacyLongField
- field8 = new LegacyLongField("field8", 0L, storedLong8),
- field6 = new LegacyLongField("field6", 0L, storedLong6),
- field4 = new LegacyLongField("field4", 0L, storedLong4),
- field2 = new LegacyLongField("field2", 0L, storedLong2);
-
- Document doc = new Document();
- // add fields, that have a distance to test general functionality
- doc.add(field8); doc.add(field6); doc.add(field4); doc.add(field2);
-
- // Add a series of noDocs docs with increasing long values, by updating the fields
- for (int l=0; l<noDocs; l++) {
- long val=distance*l+startOffset;
- field8.setLongValue(val);
- field6.setLongValue(val);
- field4.setLongValue(val);
- field2.setLongValue(val);
-
- val=l-(noDocs/2);
- writer.addDocument(doc);
- }
- Map<String,Type> map = new HashMap<>();
- map.put("field2", Type.LEGACY_LONG);
- map.put("field4", Type.LEGACY_LONG);
- map.put("field6", Type.LEGACY_LONG);
- map.put("field8", Type.LEGACY_LONG);
- reader = UninvertingReader.wrap(writer.getReader(), map);
- searcher=newSearcher(reader);
- writer.close();
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- searcher = null;
- TestUtil.checkReader(reader);
- reader.close();
- reader = null;
- directory.close();
- directory = null;
- }
-
- private void testSorting(int precisionStep) throws Exception {
- String field="field"+precisionStep;
- // 10 random tests, the index order is ascending,
- // so using a reverse sort field should retun descending documents
- int num = TestUtil.nextInt(random(), 10, 20);
- for (int i = 0; i < num; i++) {
- long lower=(long)(random().nextDouble()*noDocs*distance)+startOffset;
- long upper=(long)(random().nextDouble()*noDocs*distance)+startOffset;
- if (lower>upper) {
- long a=lower; lower=upper; upper=a;
- }
- Query tq= LegacyNumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, true);
- TopDocs topDocs = searcher.search(tq, noDocs, new Sort(new SortField(field, SortField.Type.LONG, true)));
- if (topDocs.totalHits==0) continue;
- ScoreDoc[] sd = topDocs.scoreDocs;
- assertNotNull(sd);
- long last=searcher.doc(sd[0].doc).getField(field).numericValue().longValue();
- for (int j=1; j<sd.length; j++) {
- long act=searcher.doc(sd[j].doc).getField(field).numericValue().longValue();
- assertTrue("Docs should be sorted backwards", last>act );
- last=act;
- }
- }
- }
-
- @Test
- public void testSorting_8bit() throws Exception {
- testSorting(8);
- }
-
- @Test
- public void testSorting_6bit() throws Exception {
- testSorting(6);
- }
-
- @Test
- public void testSorting_4bit() throws Exception {
- testSorting(4);
- }
-
- @Test
- public void testSorting_2bit() throws Exception {
- testSorting(2);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java
deleted file mode 100644
index 0a1cf3d..0000000
--- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.uninverting;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.Collections;
-
-import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.document.LegacyLongField;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.uninverting.UninvertingReader.Type;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.LegacyNumericUtils;
-import org.apache.lucene.util.TestUtil;
-
-public class TestUninvertingReader extends LuceneTestCase {
-
- public void testSortedSetInteger() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
-
- Document doc = new Document();
- doc.add(new LegacyIntField("foo", 5, Field.Store.NO));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new LegacyIntField("foo", 5, Field.Store.NO));
- doc.add(new LegacyIntField("foo", -3, Field.Store.NO));
- iw.addDocument(doc);
-
- iw.forceMerge(1);
- iw.close();
-
- DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
- Collections.singletonMap("foo", Type.SORTED_SET_INTEGER));
- LeafReader ar = ir.leaves().get(0).reader();
- SortedSetDocValues v = ar.getSortedSetDocValues("foo");
- assertEquals(2, v.getValueCount());
-
- v.setDocument(0);
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- v.setDocument(1);
- assertEquals(0, v.nextOrd());
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- BytesRef value = v.lookupOrd(0);
- assertEquals(-3, LegacyNumericUtils.prefixCodedToInt(value));
-
- value = v.lookupOrd(1);
- assertEquals(5, LegacyNumericUtils.prefixCodedToInt(value));
- TestUtil.checkReader(ir);
- ir.close();
- dir.close();
- }
-
- public void testSortedSetFloat() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
-
- Document doc = new Document();
- doc.add(new LegacyIntField("foo", Float.floatToRawIntBits(5f), Field.Store.NO));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new LegacyIntField("foo", Float.floatToRawIntBits(5f), Field.Store.NO));
- doc.add(new LegacyIntField("foo", Float.floatToRawIntBits(-3f), Field.Store.NO));
- iw.addDocument(doc);
-
- iw.forceMerge(1);
- iw.close();
-
- DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
- Collections.singletonMap("foo", Type.SORTED_SET_FLOAT));
- LeafReader ar = ir.leaves().get(0).reader();
-
- SortedSetDocValues v = ar.getSortedSetDocValues("foo");
- assertEquals(2, v.getValueCount());
-
- v.setDocument(0);
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- v.setDocument(1);
- assertEquals(0, v.nextOrd());
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- BytesRef value = v.lookupOrd(0);
- assertEquals(Float.floatToRawIntBits(-3f), LegacyNumericUtils.prefixCodedToInt(value));
-
- value = v.lookupOrd(1);
- assertEquals(Float.floatToRawIntBits(5f), LegacyNumericUtils.prefixCodedToInt(value));
- TestUtil.checkReader(ir);
- ir.close();
- dir.close();
- }
-
- public void testSortedSetLong() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
-
- Document doc = new Document();
- doc.add(new LegacyLongField("foo", 5, Field.Store.NO));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new LegacyLongField("foo", 5, Field.Store.NO));
- doc.add(new LegacyLongField("foo", -3, Field.Store.NO));
- iw.addDocument(doc);
-
- iw.forceMerge(1);
- iw.close();
-
- DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
- Collections.singletonMap("foo", Type.SORTED_SET_LONG));
- LeafReader ar = ir.leaves().get(0).reader();
- SortedSetDocValues v = ar.getSortedSetDocValues("foo");
- assertEquals(2, v.getValueCount());
-
- v.setDocument(0);
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- v.setDocument(1);
- assertEquals(0, v.nextOrd());
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- BytesRef value = v.lookupOrd(0);
- assertEquals(-3, LegacyNumericUtils.prefixCodedToLong(value));
-
- value = v.lookupOrd(1);
- assertEquals(5, LegacyNumericUtils.prefixCodedToLong(value));
- TestUtil.checkReader(ir);
- ir.close();
- dir.close();
- }
-
- public void testSortedSetDouble() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
-
- Document doc = new Document();
- doc.add(new LegacyLongField("foo", Double.doubleToRawLongBits(5d), Field.Store.NO));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new LegacyLongField("foo", Double.doubleToRawLongBits(5d), Field.Store.NO));
- doc.add(new LegacyLongField("foo", Double.doubleToRawLongBits(-3d), Field.Store.NO));
- iw.addDocument(doc);
-
- iw.forceMerge(1);
- iw.close();
-
- DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
- Collections.singletonMap("foo", Type.SORTED_SET_DOUBLE));
- LeafReader ar = ir.leaves().get(0).reader();
- SortedSetDocValues v = ar.getSortedSetDocValues("foo");
- assertEquals(2, v.getValueCount());
-
- v.setDocument(0);
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- v.setDocument(1);
- assertEquals(0, v.nextOrd());
- assertEquals(1, v.nextOrd());
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
-
- BytesRef value = v.lookupOrd(0);
- assertEquals(Double.doubleToRawLongBits(-3d), LegacyNumericUtils.prefixCodedToLong(value));
-
- value = v.lookupOrd(1);
- assertEquals(Double.doubleToRawLongBits(5d), LegacyNumericUtils.prefixCodedToLong(value));
- TestUtil.checkReader(ir);
- ir.close();
- dir.close();
- }
-
-
- /** Tests {@link Type#SORTED_SET_INTEGER} using Integer based fields, with and w/o precision steps */
- public void testSortedSetIntegerManyValues() throws IOException {
- final Directory dir = newDirectory();
- final IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
-
- final FieldType NO_TRIE_TYPE = new FieldType(LegacyIntField.TYPE_NOT_STORED);
- NO_TRIE_TYPE.setNumericPrecisionStep(Integer.MAX_VALUE);
-
- final Map<String,Type> UNINVERT_MAP = new LinkedHashMap<String,Type>();
- UNINVERT_MAP.put("notrie_single", Type.SORTED_SET_INTEGER);
- UNINVERT_MAP.put("notrie_multi", Type.SORTED_SET_INTEGER);
- UNINVERT_MAP.put("trie_single", Type.SORTED_SET_INTEGER);
- UNINVERT_MAP.put("trie_multi", Type.SORTED_SET_INTEGER);
- final Set<String> MULTI_VALUES = new LinkedHashSet<String>();
- MULTI_VALUES.add("trie_multi");
- MULTI_VALUES.add("notrie_multi");
-
-
- final int NUM_DOCS = TestUtil.nextInt(random(), 200, 1500);
- final int MIN = TestUtil.nextInt(random(), 10, 100);
- final int MAX = MIN + TestUtil.nextInt(random(), 10, 100);
- final long EXPECTED_VALSET_SIZE = 1 + MAX - MIN;
-
- { // (at least) one doc should have every value, so that at least one segment has every value
- final Document doc = new Document();
- for (int i = MIN; i <= MAX; i++) {
- doc.add(new LegacyIntField("trie_multi", i, Field.Store.NO));
- doc.add(new LegacyIntField("notrie_multi", i, NO_TRIE_TYPE));
- }
- iw.addDocument(doc);
- }
-
- // now add some more random docs (note: starting at i=1 because of previously added doc)
- for (int i = 1; i < NUM_DOCS; i++) {
- final Document doc = new Document();
- if (0 != TestUtil.nextInt(random(), 0, 9)) {
- int val = TestUtil.nextInt(random(), MIN, MAX);
- doc.add(new LegacyIntField("trie_single", val, Field.Store.NO));
- doc.add(new LegacyIntField("notrie_single", val, NO_TRIE_TYPE));
- }
- if (0 != TestUtil.nextInt(random(), 0, 9)) {
- int numMulti = atLeast(1);
- while (0 < numMulti--) {
- int val = TestUtil.nextInt(random(), MIN, MAX);
- doc.add(new LegacyIntField("trie_multi", val, Field.Store.NO));
- doc.add(new LegacyIntField("notrie_multi", val, NO_TRIE_TYPE));
- }
- }
- iw.addDocument(doc);
- }
-
- iw.close();
-
- final DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), UNINVERT_MAP);
- TestUtil.checkReader(ir);
-
- final int NUM_LEAVES = ir.leaves().size();
-
- // check the leaves: no more then total set size
- for (LeafReaderContext rc : ir.leaves()) {
- final LeafReader ar = rc.reader();
- for (String f : UNINVERT_MAP.keySet()) {
- final SortedSetDocValues v = DocValues.getSortedSet(ar, f);
- final long valSetSize = v.getValueCount();
- assertTrue(f + ": Expected no more then " + EXPECTED_VALSET_SIZE + " values per segment, got " +
- valSetSize + " from: " + ar.toString(),
- valSetSize <= EXPECTED_VALSET_SIZE);
-
- if (1 == NUM_LEAVES && MULTI_VALUES.contains(f)) {
- // tighter check on multi fields in single segment index since we know one doc has all of them
- assertEquals(f + ": Single segment LeafReader's value set should have had exactly expected size",
- EXPECTED_VALSET_SIZE, valSetSize);
- }
- }
- }
-
- // check the composite of all leaves: exact expectation of set size
- final LeafReader composite = SlowCompositeReaderWrapper.wrap(ir);
- TestUtil.checkReader(composite);
-
- for (String f : MULTI_VALUES) {
- final SortedSetDocValues v = composite.getSortedSetDocValues(f);
- final long valSetSize = v.getValueCount();
- assertEquals(f + ": Composite reader value set should have had exactly expected size",
- EXPECTED_VALSET_SIZE, valSetSize);
- }
-
- ir.close();
- dir.close();
- }
-
- public void testSortedSetEmptyIndex() throws IOException {
- final Directory dir = newDirectory();
- final IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
- iw.close();
-
- final Map<String,Type> UNINVERT_MAP = new LinkedHashMap<String,Type>();
- for (Type t : EnumSet.allOf(Type.class)) {
- UNINVERT_MAP.put(t.name(), t);
- }
-
- final DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), UNINVERT_MAP);
- TestUtil.checkReader(ir);
-
- final LeafReader composite = SlowCompositeReaderWrapper.wrap(ir);
- TestUtil.checkReader(composite);
-
- for (String f : UNINVERT_MAP.keySet()) {
- // check the leaves
- // (normally there are none for an empty index, so this is really just future
- // proofing in case that changes for some reason)
- for (LeafReaderContext rc : ir.leaves()) {
- final LeafReader ar = rc.reader();
- assertNull(f + ": Expected no doc values from empty index (leaf)",
- ar.getSortedSetDocValues(f));
- }
-
- // check the composite
- assertNull(f + ": Expected no doc values from empty index (composite)",
- composite.getSortedSetDocValues(f));
-
- }
-
- ir.close();
- dir.close();
- }
-
- public void testFieldInfos() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
-
- Document doc = new Document();
- BytesRef idBytes = new BytesRef("id");
- doc.add(new StringField("id", idBytes, Store.YES));
- doc.add(new LegacyIntField("int", 5, Store.YES));
- doc.add(new NumericDocValuesField("dv", 5));
- doc.add(new IntPoint("dint", 5));
- doc.add(new StoredField("stored", 5)); // not indexed
- iw.addDocument(doc);
-
- iw.forceMerge(1);
- iw.close();
-
- Map<String, Type> uninvertingMap = new HashMap<>();
- uninvertingMap.put("int", Type.LEGACY_INTEGER);
- uninvertingMap.put("dv", Type.LEGACY_INTEGER);
- uninvertingMap.put("dint", Type.INTEGER_POINT);
-
- DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
- uninvertingMap);
- LeafReader leafReader = ir.leaves().get(0).reader();
-
- FieldInfo intFInfo = leafReader.getFieldInfos().fieldInfo("int");
- assertEquals(DocValuesType.NUMERIC, intFInfo.getDocValuesType());
- assertEquals(0, intFInfo.getPointDimensionCount());
- assertEquals(0, intFInfo.getPointNumBytes());
-
- FieldInfo dintFInfo = leafReader.getFieldInfos().fieldInfo("dint");
- assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType());
- assertEquals(1, dintFInfo.getPointDimensionCount());
- assertEquals(4, dintFInfo.getPointNumBytes());
-
- FieldInfo dvFInfo = leafReader.getFieldInfos().fieldInfo("dv");
- assertEquals(DocValuesType.NUMERIC, dvFInfo.getDocValuesType());
-
- FieldInfo storedFInfo = leafReader.getFieldInfos().fieldInfo("stored");
- assertEquals(DocValuesType.NONE, storedFInfo.getDocValuesType());
-
- TestUtil.checkReader(ir);
- ir.close();
- dir.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
index 4cfb287..b186754 100644
--- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
+++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
@@ -36,12 +36,12 @@ import org.apache.lucene.search.DocValuesRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
+import org.apache.solr.uninverting.UninvertingReader.Type;
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RuleBasedCollator;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/baa5344e/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
index c38d1b5..76b38fa 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
@@ -26,16 +26,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import com.carrotsearch.hppc.IntHashSet;
-import com.carrotsearch.hppc.IntObjectHashMap;
-import com.carrotsearch.hppc.LongHashSet;
-import com.carrotsearch.hppc.LongObjectHashMap;
-import com.carrotsearch.hppc.LongObjectMap;
-import com.carrotsearch.hppc.cursors.IntObjectCursor;
-import com.carrotsearch.hppc.cursors.LongCursor;
-import com.carrotsearch.hppc.cursors.LongObjectCursor;
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
@@ -53,7 +43,6 @@ import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
-import org.apache.solr.search.QueryWrapperFilter;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
@@ -61,7 +50,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -87,12 +75,24 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocSlice;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryWrapperFilter;
import org.apache.solr.search.SolrConstantScoreQuery;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpecParsing;
+import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
+import com.carrotsearch.hppc.IntHashSet;
+import com.carrotsearch.hppc.IntObjectHashMap;
+import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.LongObjectHashMap;
+import com.carrotsearch.hppc.LongObjectMap;
+import com.carrotsearch.hppc.cursors.IntObjectCursor;
+import com.carrotsearch.hppc.cursors.LongCursor;
+import com.carrotsearch.hppc.cursors.LongObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+
/**
* The ExpandComponent is designed to work with the CollapsingPostFilter.
* The CollapsingPostFilter collapses a result set on a field.