You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:11 UTC
[23/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNumericDocValuesUpdates.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestNumericDocValuesUpdates.cs b/src/Lucene.Net.Tests/Index/TestNumericDocValuesUpdates.cs
new file mode 100644
index 0000000..827433f
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestNumericDocValuesUpdates.cs
@@ -0,0 +1,1698 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using Lucene.Net.Codecs;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Support;
+ using NUnit.Framework;
+ using System.IO;
+ using AssertingDocValuesFormat = Lucene.Net.Codecs.asserting.AssertingDocValuesFormat;
+ using BinaryDocValuesField = BinaryDocValuesField;
+ using IBits = Lucene.Net.Util.IBits;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using Codec = Lucene.Net.Codecs.Codec;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using DocValuesFormat = Lucene.Net.Codecs.DocValuesFormat;
+ using IOUtils = Lucene.Net.Util.IOUtils;
+ using Lucene40RWCodec = Lucene.Net.Codecs.Lucene40.Lucene40RWCodec;
+ using Lucene41RWCodec = Lucene.Net.Codecs.Lucene41.Lucene41RWCodec;
+ using Lucene42RWCodec = Lucene.Net.Codecs.Lucene42.Lucene42RWCodec;
+ using Lucene45DocValuesFormat = Lucene.Net.Codecs.Lucene45.Lucene45DocValuesFormat;
+ using Lucene45RWCodec = Lucene.Net.Codecs.Lucene45.Lucene45RWCodec;
+ using Lucene46Codec = Lucene.Net.Codecs.Lucene46.Lucene46Codec;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using MockTokenizer = Lucene.Net.Analysis.MockTokenizer;
+ using NumericDocValuesField = NumericDocValuesField;
+ using SortedDocValuesField = SortedDocValuesField;
+ using SortedSetDocValuesField = SortedSetDocValuesField;
+ using Store = Field.Store;
+ using StringField = StringField;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using Attributes;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ [SuppressCodecs("Appending", "Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45")]
+ [TestFixture]
+ public class TestNumericDocValuesUpdates : LuceneTestCase
+ {
+ private Document Doc(int id)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc-" + id, Store.NO));
+ // make sure we don't set the doc's value to 0, to not confuse with a document that's missing values
+ doc.Add(new NumericDocValuesField("val", id + 1));
+ return doc;
+ }
+
+ [Test]
+ public virtual void TestUpdatesAreFlushed()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)).SetRAMBufferSizeMB(0.00000001));
+ writer.AddDocument(Doc(0)); // val=1
+ writer.AddDocument(Doc(1)); // val=2
+ writer.AddDocument(Doc(3)); // val=2
+ writer.Commit();
+ Assert.AreEqual(1, writer.FlushDeletesCount);
+ writer.UpdateNumericDocValue(new Term("id", "doc-0"), "val", 5L);
+ Assert.AreEqual(2, writer.FlushDeletesCount);
+ writer.UpdateNumericDocValue(new Term("id", "doc-1"), "val", 6L);
+ Assert.AreEqual(3, writer.FlushDeletesCount);
+ writer.UpdateNumericDocValue(new Term("id", "doc-2"), "val", 7L);
+ Assert.AreEqual(4, writer.FlushDeletesCount);
+ writer.Config.SetRAMBufferSizeMB(1000d);
+ writer.UpdateNumericDocValue(new Term("id", "doc-2"), "val", 7L);
+ Assert.AreEqual(4, writer.FlushDeletesCount);
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestSimple()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ // make sure random config doesn't flush on us
+ conf.SetMaxBufferedDocs(10);
+ conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.AddDocument(Doc(0)); // val=1
+ writer.AddDocument(Doc(1)); // val=2
+ if (Random().NextBoolean()) // randomly commit before the update is sent
+ {
+ writer.Commit();
+ }
+ writer.UpdateNumericDocValue(new Term("id", "doc-0"), "val", 2L); // doc=0, exp=2
+
+ DirectoryReader reader;
+ if (Random().NextBoolean()) // not NRT
+ {
+ writer.Dispose();
+ reader = DirectoryReader.Open(dir);
+ } // NRT
+ else
+ {
+ reader = DirectoryReader.Open(writer, true);
+ writer.Dispose();
+ }
+
+ Assert.AreEqual(1, reader.Leaves.Count);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ NumericDocValues ndv = r.GetNumericDocValues("val");
+ Assert.AreEqual(2, ndv.Get(0));
+ Assert.AreEqual(2, ndv.Get(1));
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateFewSegments()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(2); // generate few segments
+ conf.SetMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges for this test
+ IndexWriter writer = new IndexWriter(dir, conf);
+ int numDocs = 10;
+ long[] expectedValues = new long[numDocs];
+ for (int i = 0; i < numDocs; i++)
+ {
+ writer.AddDocument(Doc(i));
+ expectedValues[i] = i + 1;
+ }
+ writer.Commit();
+
+ // update few docs
+ for (int i = 0; i < numDocs; i++)
+ {
+ if (Random().NextDouble() < 0.4)
+ {
+ long value = (i + 1) * 2;
+ writer.UpdateNumericDocValue(new Term("id", "doc-" + i), "val", value);
+ expectedValues[i] = value;
+ }
+ }
+
+ DirectoryReader reader;
+ if (Random().NextBoolean()) // not NRT
+ {
+ writer.Dispose();
+ reader = DirectoryReader.Open(dir);
+ } // NRT
+ else
+ {
+ reader = DirectoryReader.Open(writer, true);
+ writer.Dispose();
+ }
+
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ NumericDocValues ndv = r.GetNumericDocValues("val");
+ Assert.IsNotNull(ndv);
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ long expected = expectedValues[i + context.DocBase];
+ long actual = ndv.Get(i);
+ Assert.AreEqual(expected, actual);
+ }
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestReopen()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.AddDocument(Doc(0));
+ writer.AddDocument(Doc(1));
+
+ bool isNRT = Random().NextBoolean();
+ DirectoryReader reader1;
+ if (isNRT)
+ {
+ reader1 = DirectoryReader.Open(writer, true);
+ }
+ else
+ {
+ writer.Commit();
+ reader1 = DirectoryReader.Open(dir);
+ }
+
+ // update doc
+ writer.UpdateNumericDocValue(new Term("id", "doc-0"), "val", 10L); // update doc-0's value to 10
+ if (!isNRT)
+ {
+ writer.Commit();
+ }
+
+ // reopen reader and assert only it sees the update
+ DirectoryReader reader2 = DirectoryReader.OpenIfChanged(reader1);
+ Assert.IsNotNull(reader2);
+ Assert.IsTrue(reader1 != reader2);
+
+ Assert.AreEqual(1, ((AtomicReader)reader1.Leaves[0].Reader).GetNumericDocValues("val").Get(0));
+ Assert.AreEqual(10, ((AtomicReader)reader2.Leaves[0].Reader).GetNumericDocValues("val").Get(0));
+
+ IOUtils.Close(writer, reader1, reader2, dir);
+ }
+
+ [Test]
+ public virtual void TestUpdatesAndDeletes()
+ {
+ // create an index with a segment with only deletes, a segment with both
+ // deletes and updates and a segment with only updates
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(10); // control segment flushing
+ conf.SetMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges for this test
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ for (int i = 0; i < 6; i++)
+ {
+ writer.AddDocument(Doc(i));
+ if (i % 2 == 1)
+ {
+ writer.Commit(); // create 2-docs segments
+ }
+ }
+
+ // delete doc-1 and doc-2
+ writer.DeleteDocuments(new Term("id", "doc-1"), new Term("id", "doc-2")); // 1st and 2nd segments
+
+ // update docs 3 and 5
+ writer.UpdateNumericDocValue(new Term("id", "doc-3"), "val", 17L);
+ writer.UpdateNumericDocValue(new Term("id", "doc-5"), "val", 17L);
+
+ DirectoryReader reader;
+ if (Random().NextBoolean()) // not NRT
+ {
+ writer.Dispose();
+ reader = DirectoryReader.Open(dir);
+ } // NRT
+ else
+ {
+ reader = DirectoryReader.Open(writer, true);
+ writer.Dispose();
+ }
+
+ AtomicReader slow = SlowCompositeReaderWrapper.Wrap(reader);
+
+ IBits liveDocs = slow.LiveDocs;
+ bool[] expectedLiveDocs = new bool[] { true, false, false, true, true, true };
+ for (int i = 0; i < expectedLiveDocs.Length; i++)
+ {
+ Assert.AreEqual(expectedLiveDocs[i], liveDocs.Get(i));
+ }
+
+ long[] expectedValues = new long[] { 1, 2, 3, 17, 5, 17 };
+ NumericDocValues ndv = slow.GetNumericDocValues("val");
+ for (int i = 0; i < expectedValues.Length; i++)
+ {
+ Assert.AreEqual(expectedValues[i], ndv.Get(i));
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdatesWithDeletes()
+ {
+ // update and delete different documents in the same commit session
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(10); // control segment flushing
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ writer.AddDocument(Doc(0));
+ writer.AddDocument(Doc(1));
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ writer.DeleteDocuments(new Term("id", "doc-0"));
+ writer.UpdateNumericDocValue(new Term("id", "doc-1"), "val", 17L);
+
+ DirectoryReader reader;
+ if (Random().NextBoolean()) // not NRT
+ {
+ writer.Dispose();
+ reader = DirectoryReader.Open(dir);
+ } // NRT
+ else
+ {
+ reader = DirectoryReader.Open(writer, true);
+ writer.Dispose();
+ }
+
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ Assert.IsFalse(r.LiveDocs.Get(0));
+ Assert.AreEqual(17, r.GetNumericDocValues("val").Get(1));
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateAndDeleteSameDocument()
+ {
+ // update and delete same document in same commit session
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(10); // control segment flushing
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ writer.AddDocument(Doc(0));
+ writer.AddDocument(Doc(1));
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ writer.DeleteDocuments(new Term("id", "doc-0"));
+ writer.UpdateNumericDocValue(new Term("id", "doc-0"), "val", 17L);
+
+ DirectoryReader reader;
+ if (Random().NextBoolean()) // not NRT
+ {
+ writer.Dispose();
+ reader = DirectoryReader.Open(dir);
+ } // NRT
+ else
+ {
+ reader = DirectoryReader.Open(writer, true);
+ writer.Dispose();
+ }
+
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ Assert.IsFalse(r.LiveDocs.Get(0));
+ Assert.AreEqual(1, r.GetNumericDocValues("val").Get(0)); // deletes are currently applied first
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestMultipleDocValuesTypes()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(10); // prevent merges
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ for (int i = 0; i < 4; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("dvUpdateKey", "dv", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", i));
+ doc.Add(new BinaryDocValuesField("bdv", new BytesRef(Convert.ToString(i))));
+ doc.Add(new SortedDocValuesField("sdv", new BytesRef(Convert.ToString(i))));
+ doc.Add(new SortedSetDocValuesField("ssdv", new BytesRef(Convert.ToString(i))));
+ doc.Add(new SortedSetDocValuesField("ssdv", new BytesRef(Convert.ToString(i * 2))));
+ writer.AddDocument(doc);
+ }
+ writer.Commit();
+
+ // update all docs' ndv field
+ writer.UpdateNumericDocValue(new Term("dvUpdateKey", "dv"), "ndv", 17L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ BinaryDocValues bdv = r.GetBinaryDocValues("bdv");
+ SortedDocValues sdv = r.GetSortedDocValues("sdv");
+ SortedSetDocValues ssdv = r.GetSortedSetDocValues("ssdv");
+ BytesRef scratch = new BytesRef();
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(17, ndv.Get(i));
+ bdv.Get(i, scratch);
+ Assert.AreEqual(new BytesRef(Convert.ToString(i)), scratch);
+ sdv.Get(i, scratch);
+ Assert.AreEqual(new BytesRef(Convert.ToString(i)), scratch);
+ ssdv.SetDocument(i);
+ long ord = ssdv.NextOrd();
+ ssdv.LookupOrd(ord, scratch);
+ Assert.AreEqual(i, Convert.ToInt32(scratch.Utf8ToString()));
+ if (i != 0)
+ {
+ ord = ssdv.NextOrd();
+ ssdv.LookupOrd(ord, scratch);
+ Assert.AreEqual(i * 2, Convert.ToInt32(scratch.Utf8ToString()));
+ }
+ Assert.AreEqual(SortedSetDocValues.NO_MORE_ORDS, ssdv.NextOrd());
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestMultipleNumericDocValues()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(10); // prevent merges
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ for (int i = 0; i < 2; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("dvUpdateKey", "dv", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv1", i));
+ doc.Add(new NumericDocValuesField("ndv2", i));
+ writer.AddDocument(doc);
+ }
+ writer.Commit();
+
+ // update all docs' ndv1 field
+ writer.UpdateNumericDocValue(new Term("dvUpdateKey", "dv"), "ndv1", 17L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ NumericDocValues ndv1 = r.GetNumericDocValues("ndv1");
+ NumericDocValues ndv2 = r.GetNumericDocValues("ndv2");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(17, ndv1.Get(i));
+ Assert.AreEqual(i, ndv2.Get(i));
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDocumentWithNoValue()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ for (int i = 0; i < 2; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("dvUpdateKey", "dv", Store.NO));
+ if (i == 0) // index only one document with value
+ {
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ }
+ writer.AddDocument(doc);
+ }
+ writer.Commit();
+
+ // update all docs' ndv field
+ writer.UpdateNumericDocValue(new Term("dvUpdateKey", "dv"), "ndv", 17L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(17, ndv.Get(i));
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUnsetValue()
+ {
+ AssumeTrue("codec does not support docsWithField", DefaultCodecSupportsDocsWithField());
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ for (int i = 0; i < 2; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc" + i, Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ writer.AddDocument(doc);
+ }
+ writer.Commit();
+
+ // unset the value of 'doc0'
+ writer.UpdateNumericDocValue(new Term("id", "doc0"), "ndv", null);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ if (i == 0)
+ {
+ Assert.AreEqual(0, ndv.Get(i));
+ }
+ else
+ {
+ Assert.AreEqual(5, ndv.Get(i));
+ }
+ }
+
+ IBits docsWithField = r.GetDocsWithField("ndv");
+ Assert.IsFalse(docsWithField.Get(0));
+ Assert.IsTrue(docsWithField.Get(1));
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUnsetAllValues()
+ {
+ AssumeTrue("codec does not support docsWithField", DefaultCodecSupportsDocsWithField());
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ for (int i = 0; i < 2; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ writer.AddDocument(doc);
+ }
+ writer.Commit();
+
+ // unset the value of 'doc'
+ writer.UpdateNumericDocValue(new Term("id", "doc"), "ndv", null);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(0, ndv.Get(i));
+ }
+
+ IBits docsWithField = r.GetDocsWithField("ndv");
+ Assert.IsFalse(docsWithField.Get(0));
+ Assert.IsFalse(docsWithField.Get(1));
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateNonNumericDocValuesField()
+ {
+ // we don't support adding new fields or updating existing non-numeric-dv
+ // fields through numeric updates
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("key", "doc", Store.NO));
+ doc.Add(new StringField("foo", "bar", Store.NO));
+ writer.AddDocument(doc); // flushed document
+ writer.Commit();
+ writer.AddDocument(doc); // in-memory document
+
+ try
+ {
+ writer.UpdateNumericDocValue(new Term("key", "doc"), "ndv", 17L);
+ Assert.Fail("should not have allowed creating new fields through update");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // ok
+ }
+
+ try
+ {
+ writer.UpdateNumericDocValue(new Term("key", "doc"), "foo", 17L);
+ Assert.Fail("should not have allowed updating an existing field to numeric-dv");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // ok
+ }
+
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDifferentDVFormatPerField()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetCodec(new Lucene46CodecAnonymousInnerClassHelper(this));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("key", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ doc.Add(new SortedDocValuesField("sorted", new BytesRef("value")));
+ writer.AddDocument(doc); // flushed document
+ writer.Commit();
+ writer.AddDocument(doc); // in-memory document
+
+ writer.UpdateNumericDocValue(new Term("key", "doc"), "ndv", 17L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+
+ AtomicReader r = SlowCompositeReaderWrapper.Wrap(reader);
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ SortedDocValues sdv = r.GetSortedDocValues("sorted");
+ BytesRef scratch = new BytesRef();
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(17, ndv.Get(i));
+ sdv.Get(i, scratch);
+ Assert.AreEqual(new BytesRef("value"), scratch);
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ private class Lucene46CodecAnonymousInnerClassHelper : Lucene46Codec
+ {
+ private readonly TestNumericDocValuesUpdates OuterInstance;
+
+ public Lucene46CodecAnonymousInnerClassHelper(TestNumericDocValuesUpdates outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ public override DocValuesFormat GetDocValuesFormatForField(string field)
+ {
+ return new Lucene45DocValuesFormat();
+ }
+ }
+
+ [Test]
+ public virtual void TestUpdateSameDocMultipleTimes()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("key", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ writer.AddDocument(doc); // flushed document
+ writer.Commit();
+ writer.AddDocument(doc); // in-memory document
+
+ writer.UpdateNumericDocValue(new Term("key", "doc"), "ndv", 17L); // update existing field
+ writer.UpdateNumericDocValue(new Term("key", "doc"), "ndv", 3L); // update existing field 2nd time in this commit
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = SlowCompositeReaderWrapper.Wrap(reader);
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(3, ndv.Get(i));
+ }
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestSegmentMerges()
+ {
+ Directory dir = NewDirectory();
+ Random random = Random();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
+
+ int docid = 0;
+ int numRounds = AtLeast(10);
+ for (int rnd = 0; rnd < numRounds; rnd++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("key", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", -1));
+ int numDocs = AtLeast(30);
+ for (int i = 0; i < numDocs; i++)
+ {
+ doc.RemoveField("id");
+ doc.Add(new StringField("id", Convert.ToString(docid++), Store.NO));
+ writer.AddDocument(doc);
+ }
+
+ long value = rnd + 1;
+ writer.UpdateNumericDocValue(new Term("key", "doc"), "ndv", value);
+
+ if (random.NextDouble() < 0.2) // randomly delete some docs
+ {
+ writer.DeleteDocuments(new Term("id", Convert.ToString(random.Next(docid))));
+ }
+
+ // randomly commit or reopen-IW (or nothing), before forceMerge
+ if (random.NextDouble() < 0.4)
+ {
+ writer.Commit();
+ }
+ else if (random.NextDouble() < 0.1)
+ {
+ writer.Dispose();
+ writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
+ }
+
+ // add another document with the current value, to be sure forceMerge has
+ // something to merge (for instance, it could be that CMS finished merging
+ // all segments down to 1 before the delete was applied, so when
+ // forceMerge is called, the index will be with one segment and deletes
+ // and some MPs might now merge it, thereby invalidating test's
+ // assumption that the reader has no deletes).
+ doc = new Document();
+ doc.Add(new StringField("id", Convert.ToString(docid++), Store.NO));
+ doc.Add(new StringField("key", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", value));
+ writer.AddDocument(doc);
+
+ writer.ForceMerge(1, true);
+ DirectoryReader reader;
+ if (random.NextBoolean())
+ {
+ writer.Commit();
+ reader = DirectoryReader.Open(dir);
+ }
+ else
+ {
+ reader = DirectoryReader.Open(writer, true);
+ }
+
+ Assert.AreEqual(1, reader.Leaves.Count);
+ AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
+ Assert.IsNull(r.LiveDocs, "index should have no deletes after forceMerge");
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ Assert.IsNotNull(ndv);
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(value, ndv.Get(i));
+ }
+ reader.Dispose();
+ }
+
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateDocumentByMultipleTerms()
+ {
+ // make sure the order of updates is respected, even when multiple terms affect same document
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("k1", "v1", Store.NO));
+ doc.Add(new StringField("k2", "v2", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ writer.AddDocument(doc); // flushed document
+ writer.Commit();
+ writer.AddDocument(doc); // in-memory document
+
+ writer.UpdateNumericDocValue(new Term("k1", "v1"), "ndv", 17L);
+ writer.UpdateNumericDocValue(new Term("k2", "v2"), "ndv", 3L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = SlowCompositeReaderWrapper.Wrap(reader);
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(3, ndv.Get(i));
+ }
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestManyReopensAndFields()
+ {
+ Directory dir = NewDirectory();
+ Random random = Random();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ LogMergePolicy lmp = NewLogMergePolicy();
+ lmp.MergeFactor = 3; // merge often
+ conf.SetMergePolicy(lmp);
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ bool isNRT = random.NextBoolean();
+ DirectoryReader reader;
+ if (isNRT)
+ {
+ reader = DirectoryReader.Open(writer, true);
+ }
+ else
+ {
+ writer.Commit();
+ reader = DirectoryReader.Open(dir);
+ }
+
+ int numFields = random.Next(4) + 3; // 3-7
+ long[] fieldValues = new long[numFields];
+ bool[] fieldHasValue = new bool[numFields];
+ Arrays.Fill(fieldHasValue, true);
+ for (int i = 0; i < fieldValues.Length; i++)
+ {
+ fieldValues[i] = 1;
+ }
+
+ int numRounds = AtLeast(15);
+ int docID = 0;
+ for (int i = 0; i < numRounds; i++)
+ {
+ int numDocs = AtLeast(5);
+ // System.out.println("[" + Thread.currentThread().getName() + "]: round=" + i + ", numDocs=" + numDocs);
+ for (int j = 0; j < numDocs; j++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc-" + docID, Store.NO));
+ doc.Add(new StringField("key", "all", Store.NO)); // update key
+ // add all fields with their current value
+ for (int f = 0; f < fieldValues.Length; f++)
+ {
+ doc.Add(new NumericDocValuesField("f" + f, fieldValues[f]));
+ }
+ writer.AddDocument(doc);
+ ++docID;
+ }
+
+ // if field's value was unset before, unset it from all new added documents too
+ for (int field = 0; field < fieldHasValue.Length; field++)
+ {
+ if (!fieldHasValue[field])
+ {
+ writer.UpdateNumericDocValue(new Term("key", "all"), "f" + field, null);
+ }
+ }
+
+ int fieldIdx = random.Next(fieldValues.Length);
+ string updateField = "f" + fieldIdx;
+ if (random.NextBoolean())
+ {
+ // System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
+ fieldHasValue[fieldIdx] = false;
+ writer.UpdateNumericDocValue(new Term("key", "all"), updateField, null);
+ }
+ else
+ {
+ fieldHasValue[fieldIdx] = true;
+ writer.UpdateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
+ // System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
+ }
+
+ if (random.NextDouble() < 0.2)
+ {
+ int deleteDoc = random.Next(docID); // might also delete an already deleted document, ok!
+ writer.DeleteDocuments(new Term("id", "doc-" + deleteDoc));
+ // System.out.println("[" + Thread.currentThread().getName() + "]: deleted document: doc-" + deleteDoc);
+ }
+
+ // verify reader
+ if (!isNRT)
+ {
+ writer.Commit();
+ }
+
+ // System.out.println("[" + Thread.currentThread().getName() + "]: reopen reader: " + reader);
+ DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader);
+ Assert.IsNotNull(newReader);
+ reader.Dispose();
+ reader = newReader;
+ // System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader);
+ Assert.IsTrue(reader.NumDocs > 0); // we delete at most one document per round
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ // System.out.println(((SegmentReader) r).getSegmentName());
+ IBits liveDocs = r.LiveDocs;
+ for (int field = 0; field < fieldValues.Length; field++)
+ {
+ string f = "f" + field;
+ NumericDocValues ndv = r.GetNumericDocValues(f);
+ IBits docsWithField = r.GetDocsWithField(f);
+ Assert.IsNotNull(ndv);
+ int maxDoc = r.MaxDoc;
+ for (int doc = 0; doc < maxDoc; doc++)
+ {
+ if (liveDocs == null || liveDocs.Get(doc))
+ {
+ // System.out.println("doc=" + (doc + context.DocBase) + " f='" + f + "' vslue=" + ndv.Get(doc));
+ if (fieldHasValue[field])
+ {
+ Assert.IsTrue(docsWithField.Get(doc));
+ Assert.AreEqual(fieldValues[field], ndv.Get(doc), "invalid value for doc=" + doc + ", field=" + f + ", reader=" + r);
+ }
+ else
+ {
+ Assert.IsFalse(docsWithField.Get(doc));
+ }
+ }
+ }
+ }
+ }
+ // System.out.println();
+ }
+
+ IOUtils.Close(writer, reader, dir);
+ }
+
+ [Test]
+ public virtual void TestUpdateSegmentWithNoDocValues()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ // prevent merges, otherwise by the time updates are applied
+ // (writer.Dispose()), the segments might have merged and that update becomes
+ // legit.
+ conf.SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ // first segment with NDV
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc0", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 3));
+ writer.AddDocument(doc);
+ doc = new Document();
+ doc.Add(new StringField("id", "doc4", Store.NO)); // document without 'ndv' field
+ writer.AddDocument(doc);
+ writer.Commit();
+
+ // second segment with no NDV
+ doc = new Document();
+ doc.Add(new StringField("id", "doc1", Store.NO));
+ writer.AddDocument(doc);
+ doc = new Document();
+ doc.Add(new StringField("id", "doc2", Store.NO)); // document that isn't updated
+ writer.AddDocument(doc);
+ writer.Commit();
+
+ // update document in the first segment - should not affect docsWithField of
+ // the document without NDV field
+ writer.UpdateNumericDocValue(new Term("id", "doc0"), "ndv", 5L);
+
+ // update document in the second segment - field should be added and we should
+ // be able to handle the other document correctly (e.g. no NPE)
+ writer.UpdateNumericDocValue(new Term("id", "doc1"), "ndv", 5L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ IBits docsWithField = r.GetDocsWithField("ndv");
+ Assert.IsNotNull(docsWithField);
+ Assert.IsTrue(docsWithField.Get(0));
+ Assert.AreEqual(5L, ndv.Get(0));
+ Assert.IsFalse(docsWithField.Get(1));
+ Assert.AreEqual(0L, ndv.Get(1));
+ }
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateSegmentWithPostingButNoDocValues()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ // prevent merges, otherwise by the time updates are applied
+ // (writer.Dispose()), the segments might have merged and that update becomes
+ // legit.
+ conf.SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ // first segment with NDV
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc0", Store.NO));
+ doc.Add(new StringField("ndv", "mock-value", Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 5));
+ writer.AddDocument(doc);
+ writer.Commit();
+
+ // second segment with no NDV
+ doc = new Document();
+ doc.Add(new StringField("id", "doc1", Store.NO));
+ doc.Add(new StringField("ndv", "mock-value", Store.NO));
+ writer.AddDocument(doc);
+ writer.Commit();
+
+ // update document in the second segment
+ writer.UpdateNumericDocValue(new Term("id", "doc1"), "ndv", 5L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(5L, ndv.Get(i));
+ }
+ }
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateNumericDVFieldWithSameNameAsPostingField()
+ {
+ // this used to fail because FieldInfos.Builder neglected to update
+ // globalFieldMaps.docValueTypes map
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("f", "mock-value", Store.NO));
+ doc.Add(new NumericDocValuesField("f", 5));
+ writer.AddDocument(doc);
+ writer.Commit();
+ writer.UpdateNumericDocValue(new Term("f", "mock-value"), "f", 17L);
+ writer.Dispose();
+
+ DirectoryReader r = DirectoryReader.Open(dir);
+ NumericDocValues ndv = ((AtomicReader)r.Leaves[0].Reader).GetNumericDocValues("f");
+ Assert.AreEqual(17, ndv.Get(0));
+ r.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateOldSegments()
+ {
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
+
+ Codec[] oldCodecs = new Codec[] { new Lucene40RWCodec(OLD_FORMAT_IMPERSONATION_IS_ACTIVE), new Lucene41RWCodec(OLD_FORMAT_IMPERSONATION_IS_ACTIVE), new Lucene42RWCodec(OLD_FORMAT_IMPERSONATION_IS_ACTIVE), new Lucene45RWCodec(OLD_FORMAT_IMPERSONATION_IS_ACTIVE) };
+ Directory dir = NewDirectory();
+
+ bool oldValue = OLD_FORMAT_IMPERSONATION_IS_ACTIVE;
+ // create a segment with an old Codec
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetCodec(oldCodecs[Random().Next(oldCodecs.Length)]);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("f", 5));
+ writer.AddDocument(doc);
+ writer.Dispose();
+
+ conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ writer = new IndexWriter(dir, conf);
+ writer.UpdateNumericDocValue(new Term("id", "doc"), "f", 4L);
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = false;
+ try
+ {
+ writer.Dispose();
+ Assert.Fail("should not have succeeded to update a segment written with an old Codec");
+ }
+#pragma warning disable 168
+ catch (System.NotSupportedException e)
+#pragma warning restore 168
+ {
+ writer.Rollback();
+ }
+ finally
+ {
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = oldValue;
+ }
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestStressMultiThreading()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ // create index
+ int numThreads = TestUtil.NextInt(Random(), 3, 6);
+ int numDocs = AtLeast(2000);
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc" + i, Store.NO));
+ double group = Random().NextDouble();
+ string g;
+ if (group < 0.1)
+ {
+ g = "g0";
+ }
+ else if (group < 0.5)
+ {
+ g = "g1";
+ }
+ else if (group < 0.8)
+ {
+ g = "g2";
+ }
+ else
+ {
+ g = "g3";
+ }
+ doc.Add(new StringField("updKey", g, Store.NO));
+ for (int j = 0; j < numThreads; j++)
+ {
+ long value = Random().Next();
+ doc.Add(new NumericDocValuesField("f" + j, value));
+ doc.Add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2
+ }
+ writer.AddDocument(doc);
+ }
+
+ CountdownEvent done = new CountdownEvent(numThreads);
+ AtomicInt32 numUpdates = new AtomicInt32(AtLeast(100));
+
+ // same thread updates a field as well as reopens
+ ThreadClass[] threads = new ThreadClass[numThreads];
+ for (int i = 0; i < threads.Length; i++)
+ {
+ string f = "f" + i;
+ string cf = "cf" + i;
+ threads[i] = new ThreadAnonymousInnerClassHelper(this, "UpdateThread-" + i, writer, numDocs, done, numUpdates, f, cf);
+ }
+
+ foreach (ThreadClass t in threads)
+ {
+ t.Start();
+ }
+ done.Wait();
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ for (int i = 0; i < numThreads; i++)
+ {
+ NumericDocValues ndv = r.GetNumericDocValues("f" + i);
+ NumericDocValues control = r.GetNumericDocValues("cf" + i);
+ IBits docsWithNdv = r.GetDocsWithField("f" + i);
+ IBits docsWithControl = r.GetDocsWithField("cf" + i);
+ IBits liveDocs = r.LiveDocs;
+ for (int j = 0; j < r.MaxDoc; j++)
+ {
+ if (liveDocs == null || liveDocs.Get(j))
+ {
+ Assert.AreEqual(docsWithNdv.Get(j), docsWithControl.Get(j));
+ if (docsWithNdv.Get(j))
+ {
+ Assert.AreEqual(control.Get(j), ndv.Get(j) * 2);
+ }
+ }
+ }
+ }
+ }
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ private class ThreadAnonymousInnerClassHelper : ThreadClass
+ {
+ private readonly TestNumericDocValuesUpdates OuterInstance;
+
+ private IndexWriter Writer;
+ private int NumDocs;
+ private CountdownEvent Done;
+ private AtomicInt32 NumUpdates;
+ private string f;
+ private string Cf;
+
+ public ThreadAnonymousInnerClassHelper(TestNumericDocValuesUpdates outerInstance, string str, IndexWriter writer, int numDocs, CountdownEvent done, AtomicInt32 numUpdates, string f, string cf)
+ : base(str)
+ {
+ this.OuterInstance = outerInstance;
+ this.Writer = writer;
+ this.NumDocs = numDocs;
+ this.Done = done;
+ this.NumUpdates = numUpdates;
+ this.f = f;
+ this.Cf = cf;
+ }
+
+ public override void Run()
+ {
+ DirectoryReader reader = null;
+ bool success = false;
+ try
+ {
+ Random random = Random();
+ while (NumUpdates.GetAndDecrement() > 0)
+ {
+ double group = random.NextDouble();
+ Term t;
+ if (group < 0.1)
+ {
+ t = new Term("updKey", "g0");
+ }
+ else if (group < 0.5)
+ {
+ t = new Term("updKey", "g1");
+ }
+ else if (group < 0.8)
+ {
+ t = new Term("updKey", "g2");
+ }
+ else
+ {
+ t = new Term("updKey", "g3");
+ }
+ // System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
+ if (random.NextBoolean()) // sometimes unset a value
+ {
+ Writer.UpdateNumericDocValue(t, f, null);
+ Writer.UpdateNumericDocValue(t, Cf, null);
+ }
+ else
+ {
+ long updValue = random.Next();
+ Writer.UpdateNumericDocValue(t, f, updValue);
+ Writer.UpdateNumericDocValue(t, Cf, updValue * 2);
+ }
+
+ if (random.NextDouble() < 0.2)
+ {
+ // delete a random document
+ int doc = random.Next(NumDocs);
+ // System.out.println("[" + Thread.currentThread().getName() + "] deleteDoc=doc" + doc);
+ Writer.DeleteDocuments(new Term("id", "doc" + doc));
+ }
+
+ if (random.NextDouble() < 0.05) // commit every 20 updates on average
+ {
+ // System.out.println("[" + Thread.currentThread().getName() + "] commit");
+ Writer.Commit();
+ }
+
+ if (random.NextDouble() < 0.1) // reopen NRT reader (apply updates), on average once every 10 updates
+ {
+ if (reader == null)
+ {
+ // System.out.println("[" + Thread.currentThread().getName() + "] open NRT");
+ reader = DirectoryReader.Open(Writer, true);
+ }
+ else
+ {
+ // System.out.println("[" + Thread.currentThread().getName() + "] reopen NRT");
+ DirectoryReader r2 = DirectoryReader.OpenIfChanged(reader, Writer, true);
+ if (r2 != null)
+ {
+ reader.Dispose();
+ reader = r2;
+ }
+ }
+ }
+ }
+ // System.out.println("[" + Thread.currentThread().getName() + "] DONE");
+ success = true;
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.Message, e);
+ }
+ finally
+ {
+ if (reader != null)
+ {
+ try
+ {
+ reader.Dispose();
+ }
+ catch (IOException e)
+ {
+ if (success) // suppress this exception only if there was another exception
+ {
+ throw new Exception(e.Message, e);
+ }
+ }
+ }
+ Done.Signal();
+ }
+ }
+ }
+
+ [Test]
+ public virtual void TestUpdateDifferentDocsInDifferentGens()
+ {
+ // update same document multiple times across generations
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMaxBufferedDocs(4);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ int numDocs = AtLeast(10);
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc" + i, Store.NO));
+ long value = Random().Next();
+ doc.Add(new NumericDocValuesField("f", value));
+ doc.Add(new NumericDocValuesField("cf", value * 2));
+ writer.AddDocument(doc);
+ }
+
+ int numGens = AtLeast(5);
+ for (int i = 0; i < numGens; i++)
+ {
+ int doc = Random().Next(numDocs);
+ Term t = new Term("id", "doc" + doc);
+ long value = Random().NextLong();
+ writer.UpdateNumericDocValue(t, "f", value);
+ writer.UpdateNumericDocValue(t, "cf", value * 2);
+ DirectoryReader reader = DirectoryReader.Open(writer, true);
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ NumericDocValues fndv = r.GetNumericDocValues("f");
+ NumericDocValues cfndv = r.GetNumericDocValues("cf");
+ for (int j = 0; j < r.MaxDoc; j++)
+ {
+ Assert.AreEqual(cfndv.Get(j), fndv.Get(j) * 2);
+ }
+ }
+ reader.Dispose();
+ }
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestChangeCodec()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMergePolicy(NoMergePolicy.COMPOUND_FILES); // disable merges to simplify test assertions.
+ conf.SetCodec(new Lucene46CodecAnonymousInnerClassHelper2(this));
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
+ Document doc = new Document();
+ doc.Add(new StringField("id", "d0", Store.NO));
+ doc.Add(new NumericDocValuesField("f1", 5L));
+ doc.Add(new NumericDocValuesField("f2", 13L));
+ writer.AddDocument(doc);
+ writer.Dispose();
+
+ // change format
+ conf.SetCodec(new Lucene46CodecAnonymousInnerClassHelper3(this));
+ writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
+ doc = new Document();
+ doc.Add(new StringField("id", "d1", Store.NO));
+ doc.Add(new NumericDocValuesField("f1", 17L));
+ doc.Add(new NumericDocValuesField("f2", 2L));
+ writer.AddDocument(doc);
+ writer.UpdateNumericDocValue(new Term("id", "d0"), "f1", 12L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader r = SlowCompositeReaderWrapper.Wrap(reader);
+ NumericDocValues f1 = r.GetNumericDocValues("f1");
+ NumericDocValues f2 = r.GetNumericDocValues("f2");
+ Assert.AreEqual(12L, f1.Get(0));
+ Assert.AreEqual(13L, f2.Get(0));
+ Assert.AreEqual(17L, f1.Get(1));
+ Assert.AreEqual(2L, f2.Get(1));
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ private class Lucene46CodecAnonymousInnerClassHelper2 : Lucene46Codec
+ {
+ private readonly TestNumericDocValuesUpdates OuterInstance;
+
+ public Lucene46CodecAnonymousInnerClassHelper2(TestNumericDocValuesUpdates outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ public override DocValuesFormat GetDocValuesFormatForField(string field)
+ {
+ return new Lucene45DocValuesFormat();
+ }
+ }
+
+ private class Lucene46CodecAnonymousInnerClassHelper3 : Lucene46Codec
+ {
+ private readonly TestNumericDocValuesUpdates OuterInstance;
+
+ public Lucene46CodecAnonymousInnerClassHelper3(TestNumericDocValuesUpdates outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ public override DocValuesFormat GetDocValuesFormatForField(string field)
+ {
+ return new AssertingDocValuesFormat();
+ }
+ }
+
+ [Test]
+ public virtual void TestAddIndexes()
+ {
+ Directory dir1 = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir1, conf);
+
+ int numDocs = AtLeast(50);
+ int numTerms = TestUtil.NextInt(Random(), 1, numDocs / 5);
+ HashSet<string> randomTerms = new HashSet<string>();
+ while (randomTerms.Count < numTerms)
+ {
+ randomTerms.Add(TestUtil.RandomSimpleString(Random()));
+ }
+
+ // create first index
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("id", RandomInts.RandomFrom(Random(), randomTerms), Store.NO));
+ doc.Add(new NumericDocValuesField("ndv", 4L));
+ doc.Add(new NumericDocValuesField("control", 8L));
+ writer.AddDocument(doc);
+ }
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ // update some docs to a random value
+ long value = Random().Next();
+ Term term = new Term("id", RandomInts.RandomFrom(Random(), randomTerms));
+ writer.UpdateNumericDocValue(term, "ndv", value);
+ writer.UpdateNumericDocValue(term, "control", value * 2);
+ writer.Dispose();
+
+ Directory dir2 = NewDirectory();
+ conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ writer = new IndexWriter(dir2, conf);
+ if (Random().NextBoolean())
+ {
+ writer.AddIndexes(dir1);
+ }
+ else
+ {
+ DirectoryReader reader = DirectoryReader.Open(dir1);
+ writer.AddIndexes(reader);
+ reader.Dispose();
+ }
+ writer.Dispose();
+
+ DirectoryReader reader_ = DirectoryReader.Open(dir2);
+ foreach (AtomicReaderContext context in reader_.Leaves)
+ {
+ AtomicReader r = context.AtomicReader;
+ NumericDocValues ndv = r.GetNumericDocValues("ndv");
+ NumericDocValues control = r.GetNumericDocValues("control");
+ for (int i = 0; i < r.MaxDoc; i++)
+ {
+ Assert.AreEqual(ndv.Get(i) * 2, control.Get(i));
+ }
+ }
+ reader_.Dispose();
+
+ IOUtils.Close(dir1, dir2);
+ }
+
+ [Test]
+ public virtual void TestDeleteUnusedUpdatesFiles()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("id", "d0", Store.NO));
+ doc.Add(new NumericDocValuesField("f", 1L));
+ writer.AddDocument(doc);
+
+ // create first gen of update files
+ writer.UpdateNumericDocValue(new Term("id", "d0"), "f", 2L);
+ writer.Commit();
+ int numFiles = dir.ListAll().Length;
+
+ DirectoryReader r = DirectoryReader.Open(dir);
+ Assert.AreEqual(2L, ((AtomicReader)r.Leaves[0].Reader).GetNumericDocValues("f").Get(0));
+ r.Dispose();
+
+ // create second gen of update files, first gen should be deleted
+ writer.UpdateNumericDocValue(new Term("id", "d0"), "f", 5L);
+ writer.Commit();
+ Assert.AreEqual(numFiles, dir.ListAll().Length);
+
+ r = DirectoryReader.Open(dir);
+ Assert.AreEqual(5L, ((AtomicReader)r.Leaves[0].Reader).GetNumericDocValues("f").Get(0));
+ r.Dispose();
+
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+#if !NETSTANDARD
+ // LUCENENET: There is no Timeout on NUnit for .NET Core.
+ [Timeout(120000)]
+#endif
+ [Test, HasTimeout]
+ public virtual void TestTonsOfUpdates()
+ {
+ // LUCENE-5248: make sure that when there are many updates, we don't use too much RAM
+ Directory dir = NewDirectory();
+ Random random = Random();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ conf.SetRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
+ conf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); // don't flush by doc
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ // test data: lots of documents (few 10Ks) and lots of update terms (few hundreds)
+ int numDocs = AtLeast(20000);
+ int numNumericFields = AtLeast(5);
+ int numTerms = TestUtil.NextInt(random, 10, 100); // terms should affect many docs
+ HashSet<string> updateTerms = new HashSet<string>();
+ while (updateTerms.Count < numTerms)
+ {
+ updateTerms.Add(TestUtil.RandomSimpleString(random));
+ }
+
+ // System.out.println("numDocs=" + numDocs + " numNumericFields=" + numNumericFields + " numTerms=" + numTerms);
+
+ // build a large index with many NDV fields and update terms
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ int numUpdateTerms = TestUtil.NextInt(random, 1, numTerms / 10);
+ for (int j = 0; j < numUpdateTerms; j++)
+ {
+ doc.Add(new StringField("upd", RandomInts.RandomFrom(random, updateTerms), Store.NO));
+ }
+ for (int j = 0; j < numNumericFields; j++)
+ {
+ long val = random.Next();
+ doc.Add(new NumericDocValuesField("f" + j, val));
+ doc.Add(new NumericDocValuesField("cf" + j, val * 2));
+ }
+ writer.AddDocument(doc);
+ }
+
+ writer.Commit(); // commit so there's something to apply to
+
+ // set to flush every 2048 bytes (approximately every 12 updates), so we get
+ // many flushes during numeric updates
+ writer.Config.SetRAMBufferSizeMB(2048.0 / 1024 / 1024);
+ int numUpdates = AtLeast(100);
+ // System.out.println("numUpdates=" + numUpdates);
+ for (int i = 0; i < numUpdates; i++)
+ {
+ int field = random.Next(numNumericFields);
+ Term updateTerm = new Term("upd", RandomInts.RandomFrom(random, updateTerms));
+ long value = random.Next();
+ writer.UpdateNumericDocValue(updateTerm, "f" + field, value);
+ writer.UpdateNumericDocValue(updateTerm, "cf" + field, value * 2);
+ }
+
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ foreach (AtomicReaderContext context in reader.Leaves)
+ {
+ for (int i = 0; i < numNumericFields; i++)
+ {
+ AtomicReader r = context.AtomicReader;
+ NumericDocValues f = r.GetNumericDocValues("f" + i);
+ NumericDocValues cf = r.GetNumericDocValues("cf" + i);
+ for (int j = 0; j < r.MaxDoc; j++)
+ {
+ Assert.AreEqual(cf.Get(j), f.Get(j) * 2, "reader=" + r + ", field=f" + i + ", doc=" + j);
+ }
+ }
+ }
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdatesOrder()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("upd", "t1", Store.NO));
+ doc.Add(new StringField("upd", "t2", Store.NO));
+ doc.Add(new NumericDocValuesField("f1", 1L));
+ doc.Add(new NumericDocValuesField("f2", 1L));
+ writer.AddDocument(doc);
+ writer.UpdateNumericDocValue(new Term("upd", "t1"), "f1", 2L); // update f1 to 2
+ writer.UpdateNumericDocValue(new Term("upd", "t1"), "f2", 2L); // update f2 to 2
+ writer.UpdateNumericDocValue(new Term("upd", "t2"), "f1", 3L); // update f1 to 3
+ writer.UpdateNumericDocValue(new Term("upd", "t2"), "f2", 3L); // update f2 to 3
+ writer.UpdateNumericDocValue(new Term("upd", "t1"), "f1", 4L); // update f1 to 4 (but not f2)
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(4, ((AtomicReader)reader.Leaves[0].Reader).GetNumericDocValues("f1").Get(0));
+ Assert.AreEqual(3, ((AtomicReader)reader.Leaves[0].Reader).GetNumericDocValues("f2").Get(0));
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateAllDeletedSegment()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("f1", 1L));
+ writer.AddDocument(doc);
+ writer.AddDocument(doc);
+ writer.Commit();
+ writer.DeleteDocuments(new Term("id", "doc")); // delete all docs in the first segment
+ writer.AddDocument(doc);
+ writer.UpdateNumericDocValue(new Term("id", "doc"), "f1", 2L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(1, reader.Leaves.Count);
+ Assert.AreEqual(2L, ((AtomicReader)reader.Leaves[0].Reader).GetNumericDocValues("f1").Get(0));
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateTwoNonexistingTerms()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.Add(new StringField("id", "doc", Store.NO));
+ doc.Add(new NumericDocValuesField("f1", 1L));
+ writer.AddDocument(doc);
+ // update w/ multiple nonexisting terms in same field
+ writer.UpdateNumericDocValue(new Term("c", "foo"), "f1", 2L);
+ writer.UpdateNumericDocValue(new Term("c", "bar"), "f1", 2L);
+ writer.Dispose();
+
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(1, reader.Leaves.Count);
+ Assert.AreEqual(1L, ((AtomicReader)reader.Leaves[0].Reader).GetNumericDocValues("f1").Get(0));
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestOmitNorms.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestOmitNorms.cs b/src/Lucene.Net.Tests/Index/TestOmitNorms.cs
new file mode 100644
index 0000000..fe0697a
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestOmitNorms.cs
@@ -0,0 +1,331 @@
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using NUnit.Framework;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Analyzer = Lucene.Net.Analysis.Analyzer;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Field = Field;
+ using FieldType = FieldType;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using TextField = TextField;
+
+ [TestFixture]
+ public class TestOmitNorms : LuceneTestCase
+ {
+ // Tests whether the DocumentWriter correctly enable the
+ // omitNorms bit in the FieldInfo
+ [Test]
+ public virtual void TestOmitNorms_Mem()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+ Document d = new Document();
+
+ // this field will have norms
+ Field f1 = NewTextField("f1", "this field has norms", Field.Store.NO);
+ d.Add(f1);
+
+ // this field will NOT have norms
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.OmitNorms = true;
+ Field f2 = NewField("f2", "this field has NO norms in all docs", customType);
+ d.Add(f2);
+
+ writer.AddDocument(d);
+ writer.ForceMerge(1);
+ // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+ // keep things constant
+ d = new Document();
+
+ // Reverse
+ d.Add(NewField("f1", "this field has norms", customType));
+
+ d.Add(NewTextField("f2", "this field has NO norms in all docs", Field.Store.NO));
+
+ writer.AddDocument(d);
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ Assert.IsTrue(fi.FieldInfo("f1").OmitsNorms, "OmitNorms field bit should be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").OmitsNorms, "OmitNorms field bit should be set.");
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ // Tests whether merging of docs that have different
+ // omitNorms for the same field works
+ [Test]
+ public virtual void TestMixedMerge()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy(2)));
+ Document d = new Document();
+
+ // this field will have norms
+ Field f1 = NewTextField("f1", "this field has norms", Field.Store.NO);
+ d.Add(f1);
+
+ // this field will NOT have norms
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.OmitNorms = true;
+ Field f2 = NewField("f2", "this field has NO norms in all docs", customType);
+ d.Add(f2);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // now we add another document which has norms for field f2 and not for f1 and verify if the SegmentMerger
+ // keep things constant
+ d = new Document();
+
+ // Reverese
+ d.Add(NewField("f1", "this field has norms", customType));
+
+ d.Add(NewTextField("f2", "this field has NO norms in all docs", Field.Store.NO));
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ Assert.IsTrue(fi.FieldInfo("f1").OmitsNorms, "OmitNorms field bit should be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").OmitsNorms, "OmitNorms field bit should be set.");
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ // Make sure first adding docs that do not omitNorms for
+ // field X, then adding docs that do omitNorms for that same
+ // field,
+ [Test]
+ public virtual void TestMixedRAM()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(2)));
+ Document d = new Document();
+
+ // this field will have norms
+ Field f1 = NewTextField("f1", "this field has norms", Field.Store.NO);
+ d.Add(f1);
+
+ // this field will NOT have norms
+
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.OmitNorms = true;
+ Field f2 = NewField("f2", "this field has NO norms in all docs", customType);
+ d.Add(f2);
+
+ for (int i = 0; i < 5; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ for (int i = 0; i < 20; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // force merge
+ writer.ForceMerge(1);
+
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ Assert.IsTrue(!fi.FieldInfo("f1").OmitsNorms, "OmitNorms field bit should not be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").OmitsNorms, "OmitNorms field bit should be set.");
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ private void AssertNoNrm(Directory dir)
+ {
+ string[] files = dir.ListAll();
+ for (int i = 0; i < files.Length; i++)
+ {
+ // TODO: this relies upon filenames
+ Assert.IsFalse(files[i].EndsWith(".nrm") || files[i].EndsWith(".len"));
+ }
+ }
+
+ // Verifies no *.nrm exists when all fields omit norms:
+ [Test]
+ public virtual void TestNoNrmFile()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy()));
+ LogMergePolicy lmp = (LogMergePolicy)writer.Config.MergePolicy;
+ lmp.MergeFactor = 2;
+ lmp.NoCFSRatio = 0.0;
+ Document d = new Document();
+
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.OmitNorms = true;
+ Field f1 = NewField("f1", "this field has no norms", customType);
+ d.Add(f1);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ writer.Commit();
+
+ AssertNoNrm(ram);
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ AssertNoNrm(ram);
+ ram.Dispose();
+ }
+
+ /// <summary>
+ /// Tests various combinations of omitNorms=true/false, the field not existing at all,
+ /// ensuring that only omitNorms is 'viral'.
+ /// Internally checks that MultiNorms.norms() is consistent (returns the same bytes)
+ /// as the fully merged equivalent.
+ /// </summary>
+ [Test]
+ public virtual void TestOmitNormsCombos()
+ {
+ // indexed with norms
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ Field norms = new Field("foo", "a", customType);
+ // indexed without norms
+ FieldType customType1 = new FieldType(TextField.TYPE_STORED);
+ customType1.OmitNorms = true;
+ Field noNorms = new Field("foo", "a", customType1);
+ // not indexed, but stored
+ FieldType customType2 = new FieldType();
+ customType2.IsStored = true;
+ Field noIndex = new Field("foo", "a", customType2);
+ // not indexed but stored, omitNorms is set
+ FieldType customType3 = new FieldType();
+ customType3.IsStored = true;
+ customType3.OmitNorms = true;
+ Field noNormsNoIndex = new Field("foo", "a", customType3);
+ // not indexed nor stored (doesnt exist at all, we index a different field instead)
+ Field emptyNorms = new Field("bar", "a", customType);
+
+ Assert.IsNotNull(GetNorms("foo", norms, norms));
+ Assert.IsNull(GetNorms("foo", norms, noNorms));
+ Assert.IsNotNull(GetNorms("foo", norms, noIndex));
+ Assert.IsNotNull(GetNorms("foo", norms, noNormsNoIndex));
+ Assert.IsNotNull(GetNorms("foo", norms, emptyNorms));
+ Assert.IsNull(GetNorms("foo", noNorms, noNorms));
+ Assert.IsNull(GetNorms("foo", noNorms, noIndex));
+ Assert.IsNull(GetNorms("foo", noNorms, noNormsNoIndex));
+ Assert.IsNull(GetNorms("foo", noNorms, emptyNorms));
+ Assert.IsNull(GetNorms("foo", noIndex, noIndex));
+ Assert.IsNull(GetNorms("foo", noIndex, noNormsNoIndex));
+ Assert.IsNull(GetNorms("foo", noIndex, emptyNorms));
+ Assert.IsNull(GetNorms("foo", noNormsNoIndex, noNormsNoIndex));
+ Assert.IsNull(GetNorms("foo", noNormsNoIndex, emptyNorms));
+ Assert.IsNull(GetNorms("foo", emptyNorms, emptyNorms));
+ }
+
+ /// <summary>
+ /// Indexes at least 1 document with f1, and at least 1 document with f2.
+ /// returns the norms for "field".
+ /// </summary>
+ internal virtual NumericDocValues GetNorms(string field, Field f1, Field f2)
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy());
+ RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, iwc);
+
+ // add f1
+ Document d = new Document();
+ d.Add(f1);
+ riw.AddDocument(d);
+
+ // add f2
+ d = new Document();
+ d.Add(f2);
+ riw.AddDocument(d);
+
+ // add a mix of f1's and f2's
+ int numExtraDocs = TestUtil.NextInt(Random(), 1, 1000);
+ for (int i = 0; i < numExtraDocs; i++)
+ {
+ d = new Document();
+ d.Add(Random().NextBoolean() ? f1 : f2);
+ riw.AddDocument(d);
+ }
+
+ IndexReader ir1 = riw.Reader;
+ // todo: generalize
+ NumericDocValues norms1 = MultiDocValues.GetNormValues(ir1, field);
+
+ // fully merge and validate MultiNorms against single segment.
+ riw.ForceMerge(1);
+ DirectoryReader ir2 = riw.Reader;
+ NumericDocValues norms2 = GetOnlySegmentReader(ir2).GetNormValues(field);
+
+ if (norms1 == null)
+ {
+ Assert.IsNull(norms2);
+ }
+ else
+ {
+ for (int docID = 0; docID < ir1.MaxDoc; docID++)
+ {
+ Assert.AreEqual(norms1.Get(docID), norms2.Get(docID));
+ }
+ }
+ ir1.Dispose();
+ ir2.Dispose();
+ riw.Dispose();
+ dir.Dispose();
+ return norms1;
+ }
+ }
+}
\ No newline at end of file