You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/28 18:06:21 UTC
svn commit: r1439506 - in /lucene/dev/branches/lucene4547/lucene:
codecs/src/test/org/apache/lucene/codecs/diskdv/
codecs/src/test/org/apache/lucene/codecs/simpletext/
core/src/test/org/apache/lucene/
core/src/test/org/apache/lucene/codecs/lucene40/ co...
Author: rmuir
Date: Mon Jan 28 17:06:21 2013
New Revision: 1439506
URL: http://svn.apache.org/viewvc?rev=1439506&view=rev
Log:
try to reduce whack-a-mole
Added:
lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/
lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene42/
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java (with props)
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (with props)
Removed:
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
Added: lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,34 @@
+package org.apache.lucene.codecs.diskdv;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.util._TestUtil;
+
+/**
+ * Tests DiskDocValuesFormat
+ */
+public class TestDiskDocValuesFormat extends BaseDocValuesFormatTestCase {
+ private final Codec codec = _TestUtil.alwaysDocValuesFormat(new DiskDocValuesFormat());
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,33 @@
+package org.apache.lucene.codecs.simpletext;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.BaseDocValuesFormatTestCase;
+
+/**
+ * Tests SimpleTextDocValuesFormat
+ */
+public class TestSimpleTextDocValuesFormat extends BaseDocValuesFormatTestCase {
+ private final Codec codec = new SimpleTextCodec();
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,34 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.BaseDocValuesFormatTestCase;
+
+/**
+ * Tests Lucene40DocValuesFormat
+ */
+public class TestLucene40DocValuesFormat extends BaseDocValuesFormatTestCase {
+ private final Codec codec = new Lucene40RWCodec();
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,33 @@
+package org.apache.lucene.codecs.lucene42;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.BaseDocValuesFormatTestCase;
+
+/**
+ * Tests Lucene42DocValuesFormat
+ */
+public class TestLucene42DocValuesFormat extends BaseDocValuesFormatTestCase {
+ private final Codec codec = new Lucene42Codec();
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,122 @@
+package org.apache.lucene.codecs.perfield;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Random;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.RandomCodec;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Basic tests of PerFieldDocValuesFormat
+ */
+public class TestPerFieldDocValuesFormat extends BaseDocValuesFormatTestCase {
+ private Codec codec;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ codec = new RandomCodec(new Random(random().nextLong()), Collections.<String>emptySet());
+ }
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+
+ // just a simple trivial test
+ // nocommit: if we are going to pass down suffixes to segmentread/writestate,
+ // then they should be respected by *all* codec apis!
+ public void testTwoFieldsTwoFormats() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ final DocValuesFormat fast = DocValuesFormat.forName("Lucene42");
+ final DocValuesFormat slow = DocValuesFormat.forName("SimpleText");
+ iwc.setCodec(new Lucene42Codec() {
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ if ("dv1".equals(field)) {
+ return fast;
+ } else {
+ return slow;
+ }
+ }
+ });
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 5));
+ doc.add(new BinaryDocValuesField("dv2", new BytesRef("hello world")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ BytesRef scratch = new BytesRef();
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
+ assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
+ dv2.get(hits.scoreDocs[i].doc, scratch);
+ assertEquals(new BytesRef("hello world"), scratch);
+ }
+
+ ireader.close();
+ directory.close();
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,31 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.codecs.Codec;
+
+/** Tests the codec configuration defined by LuceneTestCase randomly
+ * (typically a mix across different fields).
+ */
+public class TestDocValuesFormat extends BaseDocValuesFormatTestCase {
+
+ @Override
+ protected Codec getCodec() {
+ return Codec.getDefault();
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1439506&r1=1439505&r2=1439506&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Mon Jan 28 17:06:21 2013
@@ -32,6 +32,7 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
@@ -1081,6 +1082,119 @@ public class TestDocValuesIndexing exten
w.close();
dir.close();
}
+
+ public void testAddSortedTwice() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo!")));
+ doc.add(new SortedDocValuesField("dv", new BytesRef("bar!")));
+ try {
+ iwriter.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+
+ iwriter.close();
+ directory.close();
+ }
+
+ public void testAddBinaryTwice() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
+ try {
+ iwriter.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+
+ iwriter.close();
+ directory.close();
+ }
+
+ public void testAddNumericTwice() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 1));
+ doc.add(new NumericDocValuesField("dv", 2));
+ try {
+ iwriter.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+
+ iwriter.close();
+ directory.close();
+ }
+
+ public void testTooLargeBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ byte bytes[] = new byte[100000];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new BinaryDocValuesField("dv", b));
+ try {
+ iwriter.addDocument(doc);
+ fail("did not get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ iwriter.close();
+
+ directory.close();
+ }
+
+ public void testTooLargeSortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ byte bytes[] = new byte[100000];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new SortedDocValuesField("dv", b));
+ try {
+ iwriter.addDocument(doc);
+ fail("did not get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ iwriter.close();
+ directory.close();
+ }
// Two documents across segments
public void testMixedTypesDifferentSegments() throws Exception {
Added: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1439506&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (added)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Mon Jan 28 17:06:21 2013
@@ -0,0 +1,971 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Abstract class to do basic tests for a docvalues format.
+ * NOTE: This test focuses on the docvalues impl, nothing else.
+ * The [stretch] goal is for this test to be
+ * so thorough in testing a new DocValuesFormat that if this
+ * test passes, then all Lucene/Solr tests should also pass. Ie,
+ * if there is some bug in a given DocValuesFormat that this
+ * test fails to catch then this test needs to be improved! */
+public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
+
+ /** Returns the codec to run tests against */
+ protected abstract Codec getCodec();
+
+ public void testOneNumber() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv", 5));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
+ assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testOneFloat() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new FloatDocValuesField("dv", 5.7f));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
+ assertEquals(Float.floatToRawIntBits(5.7f), dv.get(hits.scoreDocs[i].doc));
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testTwoNumbers() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 5));
+ doc.add(new NumericDocValuesField("dv2", 17));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
+ assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ dv = ireader.leaves().get(0).reader().getNumericDocValues("dv2");
+ assertEquals(17, dv.get(hits.scoreDocs[i].doc));
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testTwoFieldsMixed() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new NumericDocValuesField("dv1", 5));
+ doc.add(new BinaryDocValuesField("dv2", new BytesRef("hello world")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ BytesRef scratch = new BytesRef();
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
+ assertEquals(5, dv.get(hits.scoreDocs[i].doc));
+ BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
+ dv2.get(hits.scoreDocs[i].doc, scratch);
+ assertEquals(new BytesRef("hello world"), scratch);
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testThreeFieldsMixed() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new SortedDocValuesField("dv1", new BytesRef("hello hello")));
+ doc.add(new NumericDocValuesField("dv2", 5));
+ doc.add(new BinaryDocValuesField("dv3", new BytesRef("hello world")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ BytesRef scratch = new BytesRef();
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv1");
+ int ord = dv.getOrd(0);
+ dv.lookupOrd(ord, scratch);
+ assertEquals(new BytesRef("hello hello"), scratch);
+ NumericDocValues dv2 = ireader.leaves().get(0).reader().getNumericDocValues("dv2");
+ assertEquals(5, dv2.get(hits.scoreDocs[i].doc));
+ BinaryDocValues dv3 = ireader.leaves().get(0).reader().getBinaryDocValues("dv3");
+ dv3.get(hits.scoreDocs[i].doc, scratch);
+ assertEquals(new BytesRef("hello world"), scratch);
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testThreeFieldsMixed2() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new BinaryDocValuesField("dv1", new BytesRef("hello world")));
+ doc.add(new SortedDocValuesField("dv2", new BytesRef("hello hello")));
+ doc.add(new NumericDocValuesField("dv3", 5));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ BytesRef scratch = new BytesRef();
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv2");
+ int ord = dv.getOrd(0);
+ dv.lookupOrd(ord, scratch);
+ assertEquals(new BytesRef("hello hello"), scratch);
+ NumericDocValues dv2 = ireader.leaves().get(0).reader().getNumericDocValues("dv3");
+ assertEquals(5, dv2.get(hits.scoreDocs[i].doc));
+ BinaryDocValues dv3 = ireader.leaves().get(0).reader().getBinaryDocValues("dv1");
+ dv3.get(hits.scoreDocs[i].doc, scratch);
+ assertEquals(new BytesRef("hello world"), scratch);
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testTwoDocumentsNumeric() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 1));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 2));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
+ assertEquals(1, dv.get(0));
+ assertEquals(2, dv.get(1));
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testTwoDocumentsMerged() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(newField("id", "0", StringField.TYPE_STORED));
+ doc.add(new NumericDocValuesField("dv", -10));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ doc = new Document();
+ doc.add(newField("id", "1", StringField.TYPE_STORED));
+ doc.add(new NumericDocValuesField("dv", 99));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
+ for(int i=0;i<2;i++) {
+ StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
+ long expected;
+ if (doc2.get("id").equals("0")) {
+ expected = -10;
+ } else {
+ expected = 99;
+ }
+ assertEquals(expected, dv.get(i));
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testBigNumericRange() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", Long.MIN_VALUE));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("dv", Long.MAX_VALUE));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
+ assertEquals(Long.MIN_VALUE, dv.get(0));
+ assertEquals(Long.MAX_VALUE, dv.get(1));
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testBigNumericRange2() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", -8841491950446638677L));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 9062230939892376225L));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv");
+ assertEquals(-8841491950446638677L, dv.get(0));
+ assertEquals(9062230939892376225L, dv.get(1));
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("hello world")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ BytesRef scratch = new BytesRef();
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ dv.get(hits.scoreDocs[i].doc, scratch);
+ assertEquals(new BytesRef("hello world"), scratch);
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testBytesTwoDocumentsMerged() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(newField("id", "0", StringField.TYPE_STORED));
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("hello world 1")));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ doc = new Document();
+ doc.add(newField("id", "1", StringField.TYPE_STORED));
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("hello 2")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ for(int i=0;i<2;i++) {
+ StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
+ String expected;
+ if (doc2.get("id").equals("0")) {
+ expected = "hello world 1";
+ } else {
+ expected = "hello 2";
+ }
+ dv.get(i, scratch);
+ assertEquals(expected, scratch.utf8ToString());
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testSortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
+ String text = "This is the text to be indexed. " + longTerm;
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ IndexSearcher isearcher = new IndexSearcher(ireader);
+
+ assertEquals(1, isearcher.search(new TermQuery(new Term("fieldname", longTerm)), 1).totalHits);
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = isearcher.search(query, null, 1);
+ assertEquals(1, hits.totalHits);
+ BytesRef scratch = new BytesRef();
+ // Iterate through the results:
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ dv.lookupOrd(dv.getOrd(hits.scoreDocs[i].doc), scratch);
+ assertEquals(new BytesRef("hello world"), scratch);
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testSortedBytesTwoDocuments() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 1")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 2")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.lookupOrd(dv.getOrd(0), scratch);
+ assertEquals("hello world 1", scratch.utf8ToString());
+ dv.lookupOrd(dv.getOrd(1), scratch);
+ assertEquals("hello world 2", scratch.utf8ToString());
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testSortedBytesThreeDocuments() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 1")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 2")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 1")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ assertEquals(2, dv.getValueCount());
+ BytesRef scratch = new BytesRef();
+ assertEquals(0, dv.getOrd(0));
+ dv.lookupOrd(0, scratch);
+ assertEquals("hello world 1", scratch.utf8ToString());
+ assertEquals(1, dv.getOrd(1));
+ dv.lookupOrd(1, scratch);
+ assertEquals("hello world 2", scratch.utf8ToString());
+ assertEquals(0, dv.getOrd(2));
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testSortedBytesTwoDocumentsMerged() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(newField("id", "0", StringField.TYPE_STORED));
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 1")));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ doc = new Document();
+ doc.add(newField("id", "1", StringField.TYPE_STORED));
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 2")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ assertEquals(2, dv.getValueCount()); // 2 ords
+ BytesRef scratch = new BytesRef();
+ dv.lookupOrd(0, scratch);
+ assertEquals(new BytesRef("hello world 1"), scratch);
+ dv.lookupOrd(1, scratch);
+ assertEquals(new BytesRef("hello world 2"), scratch);
+ for(int i=0;i<2;i++) {
+ StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
+ String expected;
+ if (doc2.get("id").equals("0")) {
+ expected = "hello world 1";
+ } else {
+ expected = "hello world 2";
+ }
+ dv.lookupOrd(dv.getOrd(i), scratch);
+ assertEquals(expected, scratch.utf8ToString());
+ }
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testBytesWithNewline() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("hello\nworld\r1")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.get(0, scratch);
+ assertEquals(new BytesRef("hello\nworld\r1"), scratch);
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testMissingSortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("hello world 2")));
+ iwriter.addDocument(doc);
+ // 2nd doc missing the DV field
+ iwriter.addDocument(new Document());
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.lookupOrd(dv.getOrd(0), scratch);
+ assertEquals(new BytesRef("hello world 2"), scratch);
+ dv.lookupOrd(dv.getOrd(1), scratch);
+ assertEquals(new BytesRef(""), scratch);
+ ireader.close();
+ directory.close();
+ }
+
+ public void testEmptySortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ assertEquals(0, dv.getOrd(0));
+ assertEquals(0, dv.getOrd(1));
+ dv.lookupOrd(dv.getOrd(0), scratch);
+ assertEquals("", scratch.utf8ToString());
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testEmptyBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.get(0, scratch);
+ assertEquals("", scratch.utf8ToString());
+ dv.get(1, scratch);
+ assertEquals("", scratch.utf8ToString());
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testVeryLargeButLegalBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ byte bytes[] = new byte[32766];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new BinaryDocValuesField("dv", b));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.get(0, scratch);
+ assertEquals(new BytesRef(bytes), scratch);
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testVeryLargeButLegalSortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ byte bytes[] = new byte[32766];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new SortedDocValuesField("dv", b));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.get(0, scratch);
+ assertEquals(new BytesRef(bytes), scratch);
+ ireader.close();
+ directory.close();
+ }
+
+ public void testCodecUsesOwnBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("boo!")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ byte mybytes[] = new byte[20];
+ BytesRef scratch = new BytesRef(mybytes);
+ dv.get(0, scratch);
+ assertEquals("boo!", scratch.utf8ToString());
+ assertFalse(scratch.bytes == mybytes);
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testCodecUsesOwnSortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("boo!")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ byte mybytes[] = new byte[20];
+ BytesRef scratch = new BytesRef(mybytes);
+ dv.get(0, scratch);
+ assertEquals("boo!", scratch.utf8ToString());
+ assertFalse(scratch.bytes == mybytes);
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testCodecUsesOwnBytesEachTime() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.get(0, scratch);
+ assertEquals("foo!", scratch.utf8ToString());
+
+ BytesRef scratch2 = new BytesRef();
+ dv.get(1, scratch2);
+ assertEquals("bar!", scratch2.utf8ToString());
+ // check scratch is still valid
+ assertEquals("foo!", scratch.utf8ToString());
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testCodecUsesOwnSortedBytesEachTime() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ conf.setCodec(getCodec());
+ conf.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, conf);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo!")));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("bar!")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ IndexReader ireader = DirectoryReader.open(directory); // read-only=true
+ assert ireader.leaves().size() == 1;
+ BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
+ BytesRef scratch = new BytesRef();
+ dv.get(0, scratch);
+ assertEquals("foo!", scratch.utf8ToString());
+
+ BytesRef scratch2 = new BytesRef();
+ dv.get(1, scratch2);
+ assertEquals("bar!", scratch2.utf8ToString());
+ // check scratch is still valid
+ assertEquals("foo!", scratch.utf8ToString());
+
+ ireader.close();
+ directory.close();
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1439506&r1=1439505&r2=1439506&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Mon Jan 28 17:06:21 2013
@@ -43,6 +43,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42Codec;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -707,6 +708,24 @@ public class _TestUtil {
}
};
}
+
+ /** Return a Codec that can read any of the
+ * default codecs and formats, but always writes in the specified
+ * format. */
+ public static Codec alwaysDocValuesFormat(final DocValuesFormat format) {
+ // TODO: we really need for docvalues impls etc to announce themselves
+ // (and maybe their params, too) to infostream on flush and merge.
+ // otherwise in a real debugging situation we won't know whats going on!
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("forcing docvalues format to:" + format);
+ }
+ return new Lucene42Codec() {
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return format;
+ }
+ };
+ }
// TODO: generalize all 'test-checks-for-crazy-codecs' to
// annotations (LUCENE-3489)