You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:00 UTC
[12/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Search/Similarities/TestSimilarityBase.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Search/Similarities/TestSimilarityBase.cs b/src/Lucene.Net.Tests/Search/Similarities/TestSimilarityBase.cs
new file mode 100644
index 0000000..28ef7b4
--- /dev/null
+++ b/src/Lucene.Net.Tests/Search/Similarities/TestSimilarityBase.cs
@@ -0,0 +1,651 @@
+using System;
+using System.Collections.Generic;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Search.Similarities
+{
+ using NUnit.Framework;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Codec = Lucene.Net.Codecs.Codec;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Field = Field;
+ using FieldInvertState = Lucene.Net.Index.FieldInvertState;
+ using FieldType = FieldType;
+ using IndexReader = Lucene.Net.Index.IndexReader;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter;
+ using Term = Lucene.Net.Index.Term;
+ using TextField = TextField;
+
+ /// <summary>
+ /// Tests the <seealso cref="SimilarityBase"/>-based Similarities. Contains unit tests and
+ /// integration tests for all Similarities and correctness tests for a select
+ /// few.
+ /// <p>this class maintains a list of
+ /// {@code SimilarityBase} subclasses. Each test case performs its test on all
+ /// items in the list. If a test case fails, the name of the Similarity that
+ /// caused the failure is returned as part of the assertion error message.</p>
+ /// <p>Unit testing is performed by constructing statistics manually and calling
+ /// the <seealso cref="SimilarityBase#score(BasicStats, float, float)"/> method of the
+ /// Similarities. The statistics represent corner cases of corpus distributions.
+ /// </p>
+ /// <p>For the integration tests, a small (8-document) collection is indexed. The
+ /// tests verify that for a specific query, all relevant documents are returned
+ /// in the correct order. The collection consists of two poems of English poet
+ /// <a href="http://en.wikipedia.org/wiki/William_blake">William Blake</a>.</p>
+ /// <p>Note: the list of Similarities is maintained by hand. If a new Similarity
+ /// is added to the {@code Lucene.Net.Search.Similarities} package, the
+ /// list should be updated accordingly.</p>
+ /// <p>
+ /// In the correctness tests, the score is verified against the result of manual
+ /// computation. Since it would be impossible to test all Similarities
+ /// (e.g. all possible DFR combinations, all parameter values for LM), only
+ /// the best performing setups in the original papers are verified.
+ /// </p>
+ /// </summary>
+ [TestFixture]
+ public class TestSimilarityBase : LuceneTestCase
+ {
+ private static string FIELD_BODY = "body";
+ private static string FIELD_ID = "id";
+
+ /// <summary>
+ /// The tolerance range for float equality. </summary>
+ private static float FLOAT_EPSILON = 1e-5f;
+
+ /// <summary>
+ /// The DFR basic models to test. </summary>
+ internal static BasicModel[] BASIC_MODELS = new BasicModel[] { new BasicModelBE(), new BasicModelD(), new BasicModelG(), new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), new BasicModelP() };
+
+ /// <summary>
+ /// The DFR aftereffects to test. </summary>
+ internal static AfterEffect[] AFTER_EFFECTS = new AfterEffect[] { new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() };
+
+ /// <summary>
+ /// The DFR normalizations to test. </summary>
+ internal static Normalization[] NORMALIZATIONS = new Normalization[] { new NormalizationH1(), new NormalizationH2(), new NormalizationH3(), new NormalizationZ(), new Normalization.NoNormalization() };
+
+ /// <summary>
+ /// The distributions for IB. </summary>
+ internal static Distribution[] DISTRIBUTIONS = new Distribution[] { new DistributionLL(), new DistributionSPL() };
+
+ /// <summary>
+ /// Lambdas for IB. </summary>
+ internal static Lambda[] LAMBDAS = new Lambda[] { new LambdaDF(), new LambdaTTF() };
+
+ private IndexSearcher Searcher;
+ private Directory Dir;
+ private IndexReader Reader;
+
+ /// <summary>
+ /// The list of similarities to test. </summary>
+ private IList<SimilarityBase> Sims;
+
+ [SetUp]
+ public override void SetUp()
+ {
+ base.SetUp();
+
+ Dir = NewDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), Dir, Similarity, TimeZone);
+
+ for (int i = 0; i < Docs.Length; i++)
+ {
+ Document d = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.IsIndexed = false;
+ d.Add(NewField(FIELD_ID, Convert.ToString(i), ft));
+ d.Add(NewTextField(FIELD_BODY, Docs[i], Field.Store.YES));
+ writer.AddDocument(d);
+ }
+
+ Reader = writer.Reader;
+ Searcher = NewSearcher(Reader);
+ writer.Dispose();
+
+ Sims = new List<SimilarityBase>();
+ foreach (BasicModel basicModel in BASIC_MODELS)
+ {
+ foreach (AfterEffect afterEffect in AFTER_EFFECTS)
+ {
+ foreach (Normalization normalization in NORMALIZATIONS)
+ {
+ Sims.Add(new DFRSimilarity(basicModel, afterEffect, normalization));
+ }
+ }
+ }
+ foreach (Distribution distribution in DISTRIBUTIONS)
+ {
+ foreach (Lambda lambda in LAMBDAS)
+ {
+ foreach (Normalization normalization in NORMALIZATIONS)
+ {
+ Sims.Add(new IBSimilarity(distribution, lambda, normalization));
+ }
+ }
+ }
+ Sims.Add(new LMDirichletSimilarity());
+ Sims.Add(new LMJelinekMercerSimilarity(0.1f));
+ Sims.Add(new LMJelinekMercerSimilarity(0.7f));
+ }
+
+ // ------------------------------- Unit tests --------------------------------
+
+ /// <summary>
+ /// The default number of documents in the unit tests. </summary>
+ private static int NUMBER_OF_DOCUMENTS = 100;
+
+ /// <summary>
+ /// The default total number of tokens in the field in the unit tests. </summary>
+ private static long NUMBER_OF_FIELD_TOKENS = 5000;
+
+ /// <summary>
+ /// The default average field length in the unit tests. </summary>
+ private static float AVG_FIELD_LENGTH = 50;
+
+ /// <summary>
+ /// The default document frequency in the unit tests. </summary>
+ private static int DOC_FREQ = 10;
+
+ /// <summary>
+ /// The default total number of occurrences of this term across all documents
+ /// in the unit tests.
+ /// </summary>
+ private static long TOTAL_TERM_FREQ = 70;
+
+ /// <summary>
+ /// The default tf in the unit tests. </summary>
+ private static float FREQ = 7;
+
+ /// <summary>
+ /// The default document length in the unit tests. </summary>
+ private static int DOC_LEN = 40;
+
+ /// <summary>
+ /// Creates the default statistics object that the specific tests modify. </summary>
+ private BasicStats CreateStats()
+ {
+ BasicStats stats = new BasicStats("spoof", 1);
+ stats.NumberOfDocuments = NUMBER_OF_DOCUMENTS;
+ stats.NumberOfFieldTokens = NUMBER_OF_FIELD_TOKENS;
+ stats.AvgFieldLength = AVG_FIELD_LENGTH;
+ stats.DocFreq = DOC_FREQ;
+ stats.TotalTermFreq = TOTAL_TERM_FREQ;
+ return stats;
+ }
+
+ private CollectionStatistics ToCollectionStats(BasicStats stats)
+ {
+ return new CollectionStatistics(stats.Field, stats.NumberOfDocuments, -1, stats.NumberOfFieldTokens, -1);
+ }
+
+ private TermStatistics ToTermStats(BasicStats stats)
+ {
+ return new TermStatistics(new BytesRef("spoofyText"), stats.DocFreq, stats.TotalTermFreq);
+ }
+
+ /// <summary>
+ /// The generic test core called by all unit test methods. It calls the
+ /// <seealso cref="SimilarityBase#score(BasicStats, float, float)"/> method of all
+ /// Similarities in <seealso cref="#sims"/> and checks if the score is valid; i.e. it
+ /// is a finite positive real number.
+ /// </summary>
+ private void UnitTestCore(BasicStats stats, float freq, int docLen)
+ {
+ foreach (SimilarityBase sim in Sims)
+ {
+ BasicStats realStats = (BasicStats)sim.ComputeWeight(stats.TotalBoost, ToCollectionStats(stats), ToTermStats(stats));
+ float score = sim.Score(realStats, freq, docLen);
+ float explScore = sim.Explain(realStats, 1, new Explanation(freq, "freq"), docLen).Value;
+ Assert.IsFalse(float.IsInfinity(score), "Score infinite: " + sim.ToString());
+ Assert.IsFalse(float.IsNaN(score), "Score NaN: " + sim.ToString());
+ Assert.IsTrue(score >= 0, "Score negative: " + sim.ToString());
+ Assert.AreEqual(score, explScore, FLOAT_EPSILON, "score() and explain() return different values: " + sim.ToString());
+ }
+ }
+
+ /// <summary>
+ /// Runs the unit test with the default statistics. </summary>
+ [Test]
+ public virtual void TestDefault()
+ {
+ UnitTestCore(CreateStats(), FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code numberOfDocuments = numberOfFieldTokens}.
+ /// </summary>
+ [Test]
+ public virtual void TestSparseDocuments()
+ {
+ BasicStats stats = CreateStats();
+ stats.NumberOfFieldTokens = stats.NumberOfDocuments;
+ stats.TotalTermFreq = stats.DocFreq;
+ stats.AvgFieldLength = (float)stats.NumberOfFieldTokens / stats.NumberOfDocuments;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code numberOfDocuments > numberOfFieldTokens}.
+ /// </summary>
+ [Test]
+ public virtual void TestVerySparseDocuments()
+ {
+ BasicStats stats = CreateStats();
+ stats.NumberOfFieldTokens = stats.NumberOfDocuments * 2 / 3;
+ stats.TotalTermFreq = stats.DocFreq;
+ stats.AvgFieldLength = (float)stats.NumberOfFieldTokens / stats.NumberOfDocuments;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code NumberOfDocuments = 1}.
+ /// </summary>
+ [Test]
+ public virtual void TestOneDocument()
+ {
+ BasicStats stats = CreateStats();
+ stats.NumberOfDocuments = 1;
+ stats.NumberOfFieldTokens = DOC_LEN;
+ stats.AvgFieldLength = DOC_LEN;
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = (int)FREQ;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code docFreq = numberOfDocuments}.
+ /// </summary>
+ [Test]
+ public virtual void TestAllDocumentsRelevant()
+ {
+ BasicStats stats = CreateStats();
+ float mult = (0.0f + stats.NumberOfDocuments) / stats.DocFreq;
+ stats.TotalTermFreq = (int)(stats.TotalTermFreq * mult);
+ stats.DocFreq = stats.NumberOfDocuments;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code docFreq > numberOfDocuments / 2}.
+ /// </summary>
+ [Test]
+ public virtual void TestMostDocumentsRelevant()
+ {
+ BasicStats stats = CreateStats();
+ float mult = (0.6f * stats.NumberOfDocuments) / stats.DocFreq;
+ stats.TotalTermFreq = (int)(stats.TotalTermFreq * mult);
+ stats.DocFreq = (int)(stats.NumberOfDocuments * 0.6);
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code docFreq = 1}.
+ /// </summary>
+ [Test]
+ public virtual void TestOnlyOneRelevantDocument()
+ {
+ BasicStats stats = CreateStats();
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = (int)FREQ + 3;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code totalTermFreq = numberOfFieldTokens}.
+ /// </summary>
+ [Test]
+ public virtual void TestAllTermsRelevant()
+ {
+ BasicStats stats = CreateStats();
+ stats.TotalTermFreq = stats.NumberOfFieldTokens;
+ UnitTestCore(stats, DOC_LEN, DOC_LEN);
+ stats.AvgFieldLength = DOC_LEN + 10;
+ UnitTestCore(stats, DOC_LEN, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code totalTermFreq > numberOfDocuments}.
+ /// </summary>
+ [Test]
+ public virtual void TestMoreTermsThanDocuments()
+ {
+ BasicStats stats = CreateStats();
+ stats.TotalTermFreq = stats.TotalTermFreq + stats.NumberOfDocuments;
+ UnitTestCore(stats, 2 * FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when
+ /// {@code totalTermFreq = numberOfDocuments}.
+ /// </summary>
+ [Test]
+ public virtual void TestNumberOfTermsAsDocuments()
+ {
+ BasicStats stats = CreateStats();
+ stats.TotalTermFreq = stats.NumberOfDocuments;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when {@code totalTermFreq = 1}.
+ /// </summary>
+ [Test]
+ public virtual void TestOneTerm()
+ {
+ BasicStats stats = CreateStats();
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = 1;
+ UnitTestCore(stats, 1, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when {@code totalTermFreq = freq}.
+ /// </summary>
+ [Test]
+ public virtual void TestOneRelevantDocument()
+ {
+ BasicStats stats = CreateStats();
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = (int)FREQ;
+ UnitTestCore(stats, FREQ, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when {@code numberOfFieldTokens = freq}.
+ /// </summary>
+ [Test]
+ public virtual void TestAllTermsRelevantOnlyOneDocument()
+ {
+ BasicStats stats = CreateStats();
+ stats.NumberOfDocuments = 10;
+ stats.NumberOfFieldTokens = 50;
+ stats.AvgFieldLength = 5;
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = 50;
+ UnitTestCore(stats, 50, 50);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when there is only one document with a single term
+ /// in the collection.
+ /// </summary>
+ [Test]
+ public virtual void TestOnlyOneTermOneDocument()
+ {
+ BasicStats stats = CreateStats();
+ stats.NumberOfDocuments = 1;
+ stats.NumberOfFieldTokens = 1;
+ stats.AvgFieldLength = 1;
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = 1;
+ UnitTestCore(stats, 1, 1);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when there is only one term in the field, but
+ /// more than one documents.
+ /// </summary>
+ [Test]
+ public virtual void TestOnlyOneTerm()
+ {
+ BasicStats stats = CreateStats();
+ stats.NumberOfFieldTokens = 1;
+ stats.AvgFieldLength = 1.0f / stats.NumberOfDocuments;
+ stats.DocFreq = 1;
+ stats.TotalTermFreq = 1;
+ UnitTestCore(stats, 1, DOC_LEN);
+ }
+
+ /// <summary>
+ /// Tests correct behavior when {@code avgFieldLength = docLen}.
+ /// </summary>
+ [Test]
+ public virtual void TestDocumentLengthAverage()
+ {
+ BasicStats stats = CreateStats();
+ UnitTestCore(stats, FREQ, (int)stats.AvgFieldLength);
+ }
+
+ // ---------------------------- Correctness tests ----------------------------
+
+ /// <summary>
+ /// Correctness test for the Dirichlet LM model. </summary>
+ [Test]
+ public virtual void TestLMDirichlet()
+ {
+ float p = (FREQ + 2000.0f * (TOTAL_TERM_FREQ + 1) / (NUMBER_OF_FIELD_TOKENS + 1.0f)) / (DOC_LEN + 2000.0f);
+ float a = 2000.0f / (DOC_LEN + 2000.0f);
+ float gold = (float)(Math.Log(p / (a * (TOTAL_TERM_FREQ + 1) / (NUMBER_OF_FIELD_TOKENS + 1.0f))) + Math.Log(a));
+ CorrectnessTestCore(new LMDirichletSimilarity(), gold);
+ }
+
+ /// <summary>
+ /// Correctness test for the Jelinek-Mercer LM model. </summary>
+ [Test]
+ public virtual void TestLMJelinekMercer()
+ {
+ float p = (1 - 0.1f) * FREQ / DOC_LEN + 0.1f * (TOTAL_TERM_FREQ + 1) / (NUMBER_OF_FIELD_TOKENS + 1.0f);
+ float gold = (float)(Math.Log(p / (0.1f * (TOTAL_TERM_FREQ + 1) / (NUMBER_OF_FIELD_TOKENS + 1.0f))));
+ CorrectnessTestCore(new LMJelinekMercerSimilarity(0.1f), gold);
+ }
+
+ /// <summary>
+ /// Correctness test for the LL IB model with DF-based lambda and
+ /// no normalization.
+ /// </summary>
+ [Test]
+ public virtual void TestLLForIB()
+ {
+ SimilarityBase sim = new IBSimilarity(new DistributionLL(), new LambdaDF(), new Normalization.NoNormalization());
+ CorrectnessTestCore(sim, 4.178574562072754f);
+ }
+
+ /// <summary>
+ /// Correctness test for the SPL IB model with TTF-based lambda and
+ /// no normalization.
+ /// </summary>
+ [Test]
+ public virtual void TestSPLForIB()
+ {
+ SimilarityBase sim = new IBSimilarity(new DistributionSPL(), new LambdaTTF(), new Normalization.NoNormalization());
+ CorrectnessTestCore(sim, 2.2387237548828125f);
+ }
+
+ /// <summary>
+ /// Correctness test for the PL2 DFR model. </summary>
+ [Test]
+ public virtual void TestPL2()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelP(), new AfterEffectL(), new NormalizationH2());
+ float tfn = (float)(FREQ * SimilarityBase.Log2(1 + AVG_FIELD_LENGTH / DOC_LEN)); // 8.1894750101
+ float l = 1.0f / (tfn + 1.0f); // 0.108820144666
+ float lambda = (1.0f + TOTAL_TERM_FREQ) / (1f + NUMBER_OF_DOCUMENTS); // 0.7029703
+ float p = (float)(tfn * SimilarityBase.Log2(tfn / lambda) + (lambda + 1 / (12 * tfn) - tfn) * SimilarityBase.Log2(Math.E) + 0.5 * SimilarityBase.Log2(2 * Math.PI * tfn)); // 21.065619
+ float gold = l * p; // 2.2923636
+ CorrectnessTestCore(sim, gold);
+ }
+
+ /// <summary>
+ /// Correctness test for the IneB2 DFR model. </summary>
+ [Test]
+ public virtual void TestIneB2()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());
+ CorrectnessTestCore(sim, 5.747603416442871f);
+ }
+
+ /// <summary>
+ /// Correctness test for the GL1 DFR model. </summary>
+ [Test]
+ public virtual void TestGL1()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelG(), new AfterEffectL(), new NormalizationH1());
+ CorrectnessTestCore(sim, 1.6390540599822998f);
+ }
+
+ /// <summary>
+ /// Correctness test for the BEB1 DFR model. </summary>
+ [Test]
+ public virtual void TestBEB1()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelBE(), new AfterEffectB(), new NormalizationH1());
+ float tfn = FREQ * AVG_FIELD_LENGTH / DOC_LEN; // 8.75
+ float b = (TOTAL_TERM_FREQ + 1 + 1) / ((DOC_FREQ + 1) * (tfn + 1)); // 0.67132866
+ double f = TOTAL_TERM_FREQ + 1 + tfn;
+ double n = f + NUMBER_OF_DOCUMENTS;
+ double n1 = n + f - 1; // 258.5
+ double m1 = n + f - tfn - 2; // 248.75
+ double n2 = f; // 79.75
+ double m2 = f - tfn; // 71.0
+ float be = (float)(-SimilarityBase.Log2(n - 1) - SimilarityBase.Log2(Math.E) + ((m1 + 0.5f) * SimilarityBase.Log2(n1 / m1) + (n1 - m1) * SimilarityBase.Log2(n1)) - ((m2 + 0.5f) * SimilarityBase.Log2(n2 / m2) + (n2 - m2) * SimilarityBase.Log2(n2))); // 67.26544321004599 - 91.9620374903885 - -8.924494472554715
+ // 15.7720995
+ float gold = b * be; // 10.588263
+ CorrectnessTestCore(sim, gold);
+ }
+
+ /// <summary>
+ /// Correctness test for the D DFR model (basic model only). </summary>
+ [Test]
+ public virtual void TestD()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelD(), new AfterEffect.NoAfterEffect(), new Normalization.NoNormalization());
+ double totalTermFreqNorm = TOTAL_TERM_FREQ + FREQ + 1;
+ double p = 1.0 / (NUMBER_OF_DOCUMENTS + 1); // 0.009900990099009901
+ double phi = FREQ / totalTermFreqNorm; // 0.08974358974358974
+ double D = phi * SimilarityBase.Log2(phi / p) + (1 - phi) * SimilarityBase.Log2((1 - phi) / (1 - p)); // 0.17498542370019005
+ float gold = (float)(totalTermFreqNorm * D + 0.5 * SimilarityBase.Log2(1 + 2 * Math.PI * FREQ * (1 - phi))); // 16.328257
+ CorrectnessTestCore(sim, gold);
+ }
+
+ /// <summary>
+ /// Correctness test for the In2 DFR model with no aftereffect. </summary>
+ [Test]
+ public virtual void TestIn2()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelIn(), new AfterEffect.NoAfterEffect(), new NormalizationH2());
+ float tfn = (float)(FREQ * SimilarityBase.Log2(1 + AVG_FIELD_LENGTH / DOC_LEN)); // 8.1894750101
+ float gold = (float)(tfn * SimilarityBase.Log2((NUMBER_OF_DOCUMENTS + 1) / (DOC_FREQ + 0.5))); // 26.7459577898
+ CorrectnessTestCore(sim, gold);
+ }
+
+ /// <summary>
+ /// Correctness test for the IFB DFR model with no normalization. </summary>
+ [Test]
+ public virtual void TestIFB()
+ {
+ SimilarityBase sim = new DFRSimilarity(new BasicModelIF(), new AfterEffectB(), new Normalization.NoNormalization());
+ float B = (TOTAL_TERM_FREQ + 1 + 1) / ((DOC_FREQ + 1) * (FREQ + 1)); // 0.8875
+ float IF = (float)(FREQ * SimilarityBase.Log2(1 + (NUMBER_OF_DOCUMENTS + 1) / (TOTAL_TERM_FREQ + 0.5))); // 8.97759389642
+ float gold = B * IF; // 7.96761458307
+ CorrectnessTestCore(sim, gold);
+ }
+
+ /// <summary>
+ /// The generic test core called by all correctness test methods. It calls the
+ /// <seealso cref="SimilarityBase#score(BasicStats, float, float)"/> method of all
+ /// Similarities in <seealso cref="#sims"/> and compares the score against the manually
+ /// computed {@code gold}.
+ /// </summary>
+ private void CorrectnessTestCore(SimilarityBase sim, float gold)
+ {
+ BasicStats stats = CreateStats();
+ BasicStats realStats = (BasicStats)sim.ComputeWeight(stats.TotalBoost, ToCollectionStats(stats), ToTermStats(stats));
+ float score = sim.Score(realStats, FREQ, DOC_LEN);
+ Assert.AreEqual(gold, score, FLOAT_EPSILON, sim.ToString() + " score not correct.");
+ }
+
+ // ---------------------------- Integration tests ----------------------------
+
+ /// <summary>
+ /// The "collection" for the integration tests. </summary>
+ internal string[] Docs = new string[] { "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Could frame thy fearful symmetry ?", "In what distant depths or skies Burnt the fire of thine eyes ? On what wings dare he aspire ? What the hands the seize the fire ?", "And what shoulder and what art Could twist the sinews of thy heart ? And when thy heart began to beat What dread hand ? And what dread feet ?", "What the hammer? What the chain ? In what furnace was thy brain ? What the anvil ? And what dread grasp Dare its deadly terrors clasp ?", "And when the stars threw down their spears And water'd heaven with their tear Did he smile his work to see ? Did he, who made the lamb, made thee ?", "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Dare frame thy fearful symmetry ?", "Cruelty has a human heart And jealousy a human face Terror the human form divine And Secrecy the human
dress .", "The human dress is forg'd iron The human form a fiery forge The human face a furnace seal'd The human heart its fiery gorge ." };
+
+ /// <summary>
+ /// Tests whether all similarities return three documents for the query word
+ /// "heart".
+ /// </summary>
+ [Test]
+ public virtual void TestHeartList()
+ {
+ Query q = new TermQuery(new Term(FIELD_BODY, "heart"));
+
+ foreach (SimilarityBase sim in Sims)
+ {
+ Searcher.Similarity = sim;
+ TopDocs topDocs = Searcher.Search(q, 1000);
+ Assert.AreEqual(3, topDocs.TotalHits, "Failed: " + sim.ToString());
+ }
+ }
+
+ /// <summary>
+ /// Test whether all similarities return document 3 before documents 7 and 8. </summary>
+ [Test]
+ public virtual void TestHeartRanking()
+ {
+ AssumeFalse("PreFlex codec does not support the stats necessary for this test!", "Lucene3x".Equals(Codec.Default.Name));
+
+ Query q = new TermQuery(new Term(FIELD_BODY, "heart"));
+
+ foreach (SimilarityBase sim in Sims)
+ {
+ Searcher.Similarity = sim;
+ TopDocs topDocs = Searcher.Search(q, 1000);
+ Assert.AreEqual("2", Reader.Document(topDocs.ScoreDocs[0].Doc).Get(FIELD_ID), "Failed: " + sim.ToString());
+ }
+ }
+
+ [TearDown]
+ public override void TearDown()
+ {
+ Reader.Dispose();
+ Dir.Dispose();
+ base.TearDown();
+ }
+
+ // LUCENE-5221
+ [Test]
+ public virtual void TestDiscountOverlapsBoost()
+ {
+ DefaultSimilarity expected = new DefaultSimilarity();
+ SimilarityBase actual = new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());
+ expected.DiscountOverlaps = false;
+ actual.DiscountOverlaps = false;
+ FieldInvertState state = new FieldInvertState("foo");
+ state.Length = 5;
+ state.NumOverlap = 2;
+ state.Boost = 3;
+ Assert.AreEqual(expected.ComputeNorm(state), actual.ComputeNorm(state));
+ expected.DiscountOverlaps = true;
+ actual.DiscountOverlaps = true;
+ Assert.AreEqual(expected.ComputeNorm(state), actual.ComputeNorm(state));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Search/SingleDocTestFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Search/SingleDocTestFilter.cs b/src/Lucene.Net.Tests/Search/SingleDocTestFilter.cs
new file mode 100644
index 0000000..8e404ea
--- /dev/null
+++ b/src/Lucene.Net.Tests/Search/SingleDocTestFilter.cs
@@ -0,0 +1,44 @@
+namespace Lucene.Net.Search
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using AtomicReaderContext = Lucene.Net.Index.AtomicReaderContext;
+ using IBits = Lucene.Net.Util.IBits;
+ using FixedBitSet = Lucene.Net.Util.FixedBitSet;
+
+ public class SingleDocTestFilter : Filter
+ {
+ private int Doc;
+
+ public SingleDocTestFilter(int doc)
+ {
+ this.Doc = doc;
+ }
+
+ public override DocIdSet GetDocIdSet(AtomicReaderContext context, IBits acceptDocs)
+ {
+ FixedBitSet bits = new FixedBitSet(context.Reader.MaxDoc);
+ bits.Set(Doc);
+ if (acceptDocs != null && !acceptDocs.Get(Doc))
+ {
+ bits.Clear(Doc);
+ }
+ return bits;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Search/Spans/JustCompileSearchSpans.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Search/Spans/JustCompileSearchSpans.cs b/src/Lucene.Net.Tests/Search/Spans/JustCompileSearchSpans.cs
new file mode 100644
index 0000000..f2b71ae
--- /dev/null
+++ b/src/Lucene.Net.Tests/Search/Spans/JustCompileSearchSpans.cs
@@ -0,0 +1,165 @@
+using System.Collections.Generic;
+
+namespace Lucene.Net.Search.Spans
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using AtomicReaderContext = Lucene.Net.Index.AtomicReaderContext;
+ using IBits = Lucene.Net.Util.IBits;
+ using Similarity = Lucene.Net.Search.Similarities.Similarity;
+ using Term = Lucene.Net.Index.Term;
+ using TermContext = Lucene.Net.Index.TermContext;
+
+ /// <summary>
+ /// Holds all implementations of classes in the o.a.l.s.spans package as a
+ /// back-compatibility test. It does not run any tests per-se, however if
+ /// someone adds a method to an interface or abstract method to an abstract
+ /// class, one of the implementations here will fail to compile and so we know
+ /// back-compat policy was violated.
+ /// </summary>
+ internal sealed class JustCompileSearchSpans
+ {
+ private const string UNSUPPORTED_MSG = "unsupported: used for back-compat testing only !";
+
+ internal sealed class JustCompileSpans : Spans
+ {
+ public override int Doc
+ {
+ get { throw new System.NotSupportedException(UNSUPPORTED_MSG); }
+ }
+
+ public override int End
+ {
+ get { throw new System.NotSupportedException(UNSUPPORTED_MSG); }
+ }
+
+ public override bool Next()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override bool SkipTo(int target)
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override int Start
+ {
+ get { throw new System.NotSupportedException(UNSUPPORTED_MSG); }
+ }
+
+ public override ICollection<byte[]> GetPayload()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override bool IsPayloadAvailable
+ {
+ get
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+
+ public override long GetCost()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+
+ internal sealed class JustCompileSpanQuery : SpanQuery
+ {
+ public override string Field
+ {
+ get
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+
+ public override Spans GetSpans(AtomicReaderContext context, IBits acceptDocs, IDictionary<Term, TermContext> termContexts)
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override string ToString(string field)
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+
+ internal sealed class JustCompilePayloadSpans : Spans
+ {
+ public override ICollection<byte[]> GetPayload()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override bool IsPayloadAvailable
+ {
+ get
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+
+ public override int Doc
+ {
+ get { throw new System.NotSupportedException(UNSUPPORTED_MSG); }
+ }
+
+ public override int End
+ {
+ get { throw new System.NotSupportedException(UNSUPPORTED_MSG); }
+ }
+
+ public override bool Next()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override bool SkipTo(int target)
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+
+ public override int Start
+ {
+ get { throw new System.NotSupportedException(UNSUPPORTED_MSG); }
+ }
+
+ public override long GetCost()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+
+ internal sealed class JustCompileSpanScorer : SpanScorer
+ {
+ internal JustCompileSpanScorer(Spans spans, Weight weight, Similarity.SimScorer docScorer)
+ : base(spans, weight, docScorer)
+ {
+ }
+
+ protected override bool SetFreqCurrentDoc()
+ {
+ throw new System.NotSupportedException(UNSUPPORTED_MSG);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Search/Spans/MultiSpansWrapper.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Search/Spans/MultiSpansWrapper.cs b/src/Lucene.Net.Tests/Search/Spans/MultiSpansWrapper.cs
new file mode 100644
index 0000000..2a53f3e
--- /dev/null
+++ b/src/Lucene.Net.Tests/Search/Spans/MultiSpansWrapper.cs
@@ -0,0 +1,215 @@
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Lucene.Net.Search.Spans
+{
+ using Lucene.Net.Index;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using AtomicReaderContext = Lucene.Net.Index.AtomicReaderContext;
+ using IndexReaderContext = Lucene.Net.Index.IndexReaderContext;
+ using ReaderUtil = Lucene.Net.Index.ReaderUtil;
+ using Term = Lucene.Net.Index.Term;
+ using TermContext = Lucene.Net.Index.TermContext;
+
+ ///
+ /// <summary>
+ /// A wrapper to perform span operations on a non-leaf reader context
+ /// <p>
+ /// NOTE: this should be used for testing purposes only
+ /// @lucene.internal
+ /// </summary>
+ public class MultiSpansWrapper : Spans // can't be package private due to payloads
+ {
+ private readonly SpanQuery Query;
+ private readonly IList<AtomicReaderContext> Leaves;
+ private int LeafOrd = 0;
+ private Spans Current;
+ private readonly IDictionary<Term, TermContext> TermContexts;
+ private readonly int NumLeaves;
+
+ private MultiSpansWrapper(IList<AtomicReaderContext> leaves, SpanQuery query, IDictionary<Term, TermContext> termContexts)
+ {
+ this.Query = query;
+ this.Leaves = leaves;
+ this.NumLeaves = leaves.Count;
+ this.TermContexts = termContexts;
+ }
+
+ public static Spans Wrap(IndexReaderContext topLevelReaderContext, SpanQuery query)
+ {
+ IDictionary<Term, TermContext> termContexts = new Dictionary<Term, TermContext>();
+ SortedSet<Term> terms = new SortedSet<Term>();
+ query.ExtractTerms(terms);
+ foreach (Term term in terms)
+ {
+ termContexts[term] = TermContext.Build(topLevelReaderContext, term);
+ }
+ IList<AtomicReaderContext> leaves = topLevelReaderContext.Leaves;
+ if (leaves.Count == 1)
+ {
+ AtomicReaderContext ctx = leaves[0];
+ return query.GetSpans(ctx, ((AtomicReader)ctx.Reader).LiveDocs, termContexts);
+ }
+ return new MultiSpansWrapper(leaves, query, termContexts);
+ }
+
+ public override bool Next()
+ {
+ if (LeafOrd >= NumLeaves)
+ {
+ return false;
+ }
+ if (Current == null)
+ {
+ AtomicReaderContext ctx = Leaves[LeafOrd];
+ Current = Query.GetSpans(ctx, ((AtomicReader)ctx.Reader).LiveDocs, TermContexts);
+ }
+ while (true)
+ {
+ if (Current.Next())
+ {
+ return true;
+ }
+ if (++LeafOrd < NumLeaves)
+ {
+ AtomicReaderContext ctx = Leaves[LeafOrd];
+ Current = Query.GetSpans(ctx, ((AtomicReader)ctx.Reader).LiveDocs, TermContexts);
+ }
+ else
+ {
+ Current = null;
+ break;
+ }
+ }
+ return false;
+ }
+
+ public override bool SkipTo(int target)
+ {
+ if (LeafOrd >= NumLeaves)
+ {
+ return false;
+ }
+
+ int subIndex = ReaderUtil.SubIndex(target, Leaves);
+ Debug.Assert(subIndex >= LeafOrd);
+ if (subIndex != LeafOrd)
+ {
+ AtomicReaderContext ctx = Leaves[subIndex];
+ Current = Query.GetSpans(ctx, ((AtomicReader)ctx.Reader).LiveDocs, TermContexts);
+ LeafOrd = subIndex;
+ }
+ else if (Current == null)
+ {
+ AtomicReaderContext ctx = Leaves[LeafOrd];
+ Current = Query.GetSpans(ctx, ((AtomicReader)ctx.Reader).LiveDocs, TermContexts);
+ }
+ while (true)
+ {
+ if (target < Leaves[LeafOrd].DocBase)
+ {
+ // target was in the previous slice
+ if (Current.Next())
+ {
+ return true;
+ }
+ }
+ else if (Current.SkipTo(target - Leaves[LeafOrd].DocBase))
+ {
+ return true;
+ }
+ if (++LeafOrd < NumLeaves)
+ {
+ AtomicReaderContext ctx = Leaves[LeafOrd];
+ Current = Query.GetSpans(ctx, ((AtomicReader)ctx.Reader).LiveDocs, TermContexts);
+ }
+ else
+ {
+ Current = null;
+ break;
+ }
+ }
+
+ return false;
+ }
+
+ public override int Doc
+ {
+ get
+ {
+ if (Current == null)
+ {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+ return Current.Doc + Leaves[LeafOrd].DocBase;
+ }
+ }
+
+ public override int Start
+ {
+ get
+ {
+ if (Current == null)
+ {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+ return Current.Start;
+ }
+ }
+
+ public override int End
+ {
+ get
+ {
+ if (Current == null)
+ {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+ return Current.End;
+ }
+ }
+
+ public override ICollection<byte[]> GetPayload()
+ {
+ if (Current == null)
+ {
+ return new List<byte[]>();
+ }
+ return Current.GetPayload();
+ }
+
+ public override bool IsPayloadAvailable
+ {
+ get
+ {
+ if (Current == null)
+ {
+ return false;
+ }
+ return Current.IsPayloadAvailable;
+ }
+ }
+
+ public override long GetCost()
+ {
+ return int.MaxValue; // just for tests
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Search/Spans/TestBasics.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Search/Spans/TestBasics.cs b/src/Lucene.Net.Tests/Search/Spans/TestBasics.cs
new file mode 100644
index 0000000..737aa82
--- /dev/null
+++ b/src/Lucene.Net.Tests/Search/Spans/TestBasics.cs
@@ -0,0 +1,626 @@
+using Lucene.Net.Analysis.TokenAttributes;
+using Lucene.Net.Documents;
+using Lucene.Net.Support;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Search.Spans
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Lucene.Net.Analysis;
+ using Lucene.Net.Util;
+ using NUnit.Framework;
+ using System.IO;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using English = Lucene.Net.Util.English;
+ using Field = Field;
+ using IndexReader = Lucene.Net.Index.IndexReader;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter;
+ using Term = Lucene.Net.Index.Term;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ /// <summary>
+ /// Tests basic search capabilities.
+ ///
+ /// <p>Uses a collection of 1000 documents, each the english rendition of their
+ /// document number. For example, the document numbered 333 has text "three
+ /// hundred thirty three".
+ ///
+ /// <p>Tests are each a single query, and its hits are checked to ensure that
+ /// all and only the correct documents are returned, thus providing end-to-end
+ /// testing of the indexing and search code.
+ ///
+ /// </summary>
+ [TestFixture]
+ public class TestBasics : LuceneTestCase
+ {
+ private static IndexSearcher Searcher;
+ private static IndexReader Reader;
+ private static Directory Directory;
+
+ internal sealed class SimplePayloadFilter : TokenFilter
+ {
+ internal int Pos;
+ internal readonly IPayloadAttribute PayloadAttr;
+ internal readonly ICharTermAttribute TermAttr;
+
+ public SimplePayloadFilter(TokenStream input)
+ : base(input)
+ {
+ Pos = 0;
+ PayloadAttr = input.AddAttribute<IPayloadAttribute>();
+ TermAttr = input.AddAttribute<ICharTermAttribute>();
+ }
+
+ public override bool IncrementToken()
+ {
+ if (m_input.IncrementToken())
+ {
+#pragma warning disable 612, 618
+ PayloadAttr.Payload = new BytesRef(("pos: " + Pos).GetBytes(IOUtils.CHARSET_UTF_8));
+#pragma warning restore 612, 618
+ Pos++;
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ public override void Reset()
+ {
+ base.Reset();
+ Pos = 0;
+ }
+ }
+
+ internal static Analyzer SimplePayloadAnalyzer;
+
+ /// <summary>
+ /// LUCENENET specific
+ /// Is non-static because NewIndexWriterConfig is no longer static.
+ /// </summary>
+ [OneTimeSetUp]
+ public void BeforeClass()
+ {
+ SimplePayloadAnalyzer = new AnalyzerAnonymousInnerClassHelper();
+
+ Directory = NewDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), Directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, SimplePayloadAnalyzer).SetMaxBufferedDocs(TestUtil.NextInt(Random(), 100, 1000)).SetMergePolicy(NewLogMergePolicy()));
+ //writer.infoStream = System.out;
+ for (int i = 0; i < 2000; i++)
+ {
+ Document doc = new Document();
+ doc.Add(NewTextField("field", English.IntToEnglish(i), Field.Store.YES));
+ writer.AddDocument(doc);
+ }
+ Reader = writer.Reader;
+ Searcher = NewSearcher(Reader);
+ writer.Dispose();
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ public AnalyzerAnonymousInnerClassHelper()
+ {
+ }
+
+ protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(tokenizer, new SimplePayloadFilter(tokenizer));
+ }
+ }
+
+ [OneTimeTearDown]
+ public static void AfterClass()
+ {
+ Reader.Dispose();
+ Directory.Dispose();
+ Searcher = null;
+ Reader = null;
+ Directory = null;
+ SimplePayloadAnalyzer = null;
+ }
+
+ [Test]
+ public virtual void TestTerm()
+ {
+ Query query = new TermQuery(new Term("field", "seventy"));
+ CheckHits(query, new int[] { 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777
, 1778, 1779, 1870, 1871, 1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979 });
+ }
+
+ [Test]
+ public virtual void TestTerm2()
+ {
+ Query query = new TermQuery(new Term("field", "seventish"));
+ CheckHits(query, new int[] { });
+ }
+
+ [Test]
+ public virtual void TestPhrase()
+ {
+ PhraseQuery query = new PhraseQuery();
+ query.Add(new Term("field", "seventy"));
+ query.Add(new Term("field", "seven"));
+ CheckHits(query, new int[] { 77, 177, 277, 377, 477, 577, 677, 777, 877, 977, 1077, 1177, 1277, 1377, 1477, 1577, 1677, 1777, 1877, 1977 });
+ }
+
+ [Test]
+ public virtual void TestPhrase2()
+ {
+ PhraseQuery query = new PhraseQuery();
+ query.Add(new Term("field", "seventish"));
+ query.Add(new Term("field", "sevenon"));
+ CheckHits(query, new int[] { });
+ }
+
+ [Test]
+ public virtual void TestBoolean()
+ {
+ BooleanQuery query = new BooleanQuery();
+ query.Add(new TermQuery(new Term("field", "seventy")), Occur.MUST);
+ query.Add(new TermQuery(new Term("field", "seven")), Occur.MUST);
+ CheckHits(query, new int[] { 77, 177, 277, 377, 477, 577, 677, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 877, 977, 1077, 1177, 1277, 1377, 1477, 1577, 1677, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1877, 1977 });
+ }
+
+ [Test]
+ public virtual void TestBoolean2()
+ {
+ BooleanQuery query = new BooleanQuery();
+ query.Add(new TermQuery(new Term("field", "sevento")), Occur.MUST);
+ query.Add(new TermQuery(new Term("field", "sevenly")), Occur.MUST);
+ CheckHits(query, new int[] { });
+ }
+
+ [Test]
+ public virtual void TestSpanNearExact()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "seventy"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "seven"));
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 0, true);
+ CheckHits(query, new int[] { 77, 177, 277, 377, 477, 577, 677, 777, 877, 977, 1077, 1177, 1277, 1377, 1477, 1577, 1677, 1777, 1877, 1977 });
+
+ Assert.IsTrue(Searcher.Explain(query, 77).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 977).Value > 0.0f);
+
+ QueryUtils.Check(term1);
+ QueryUtils.Check(term2);
+ QueryUtils.CheckUnequal(term1, term2);
+ }
+
+ [Test]
+ public virtual void TestSpanTermQuery()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "seventy"));
+ CheckHits(term1, new int[] { 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1170, 1270, 1370, 1470, 1570, 1670, 1770, 1870, 1970, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1671, 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1771, 1772, 1773, 1774, 1775
, 1776, 1777, 1778, 1779, 1871, 1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979 });
+ }
+
+ [Test]
+ public virtual void TestSpanNearUnordered()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "nine"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "six"));
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, false);
+
+ CheckHits(query, new int[] { 609, 629, 639, 649, 659, 669, 679, 689, 699, 906, 926, 936, 946, 956, 966, 976, 986, 996, 1609, 1629, 1639, 1649, 1659, 1669, 1679, 1689, 1699, 1906, 1926, 1936, 1946, 1956, 1966, 1976, 1986, 1996 });
+ }
+
+ [Test]
+ public virtual void TestSpanNearOrdered()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "nine"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "six"));
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ CheckHits(query, new int[] { 906, 926, 936, 946, 956, 966, 976, 986, 996, 1906, 1926, 1936, 1946, 1956, 1966, 1976, 1986, 1996 });
+ }
+
+ [Test]
+ public virtual void TestSpanNot()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "one"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "forty"));
+ SpanNotQuery query = new SpanNotQuery(near, term3);
+
+ CheckHits(query, new int[] { 801, 821, 831, 851, 861, 871, 881, 891, 1801, 1821, 1831, 1851, 1861, 1871, 1881, 1891 });
+
+ Assert.IsTrue(Searcher.Explain(query, 801).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 891).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanWithMultipleNotSingle()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "one"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "forty"));
+
+ SpanOrQuery or = new SpanOrQuery(term3);
+
+ SpanNotQuery query = new SpanNotQuery(near, or);
+
+ CheckHits(query, new int[] { 801, 821, 831, 851, 861, 871, 881, 891, 1801, 1821, 1831, 1851, 1861, 1871, 1881, 1891 });
+
+ Assert.IsTrue(Searcher.Explain(query, 801).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 891).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanWithMultipleNotMany()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "one"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "forty"));
+ SpanTermQuery term4 = new SpanTermQuery(new Term("field", "sixty"));
+ SpanTermQuery term5 = new SpanTermQuery(new Term("field", "eighty"));
+
+ SpanOrQuery or = new SpanOrQuery(term3, term4, term5);
+
+ SpanNotQuery query = new SpanNotQuery(near, or);
+
+ CheckHits(query, new int[] { 801, 821, 831, 851, 871, 891, 1801, 1821, 1831, 1851, 1871, 1891 });
+
+ Assert.IsTrue(Searcher.Explain(query, 801).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 891).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestNpeInSpanNearWithSpanNot()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "one"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery hun = new SpanTermQuery(new Term("field", "hundred"));
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "forty"));
+ SpanNearQuery exclude = new SpanNearQuery(new SpanQuery[] { hun, term3 }, 1, true);
+
+ SpanNotQuery query = new SpanNotQuery(near, exclude);
+
+ CheckHits(query, new int[] { 801, 821, 831, 851, 861, 871, 881, 891, 1801, 1821, 1831, 1851, 1861, 1871, 1881, 1891 });
+
+ Assert.IsTrue(Searcher.Explain(query, 801).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 891).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestNpeInSpanNearInSpanFirstInSpanNot()
+ {
+ int n = 5;
+ SpanTermQuery hun = new SpanTermQuery(new Term("field", "hundred"));
+ SpanTermQuery term40 = new SpanTermQuery(new Term("field", "forty"));
+ SpanTermQuery term40c = (SpanTermQuery)term40.Clone();
+
+ SpanFirstQuery include = new SpanFirstQuery(term40, n);
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { hun, term40c }, n - 1, true);
+ SpanFirstQuery exclude = new SpanFirstQuery(near, n - 1);
+ SpanNotQuery q = new SpanNotQuery(include, exclude);
+
+ CheckHits(q, new int[] { 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, 1340, 1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1840, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949 });
+ }
+
+ [Test]
+ public virtual void TestSpanNotWindowOne()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "forty"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "one"));
+ SpanNotQuery query = new SpanNotQuery(near, term3, 1, 1);
+
+ CheckHits(query, new int[] { 840, 842, 843, 844, 845, 846, 847, 848, 849, 1840, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849 });
+
+ Assert.IsTrue(Searcher.Explain(query, 840).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 1842).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanNotWindowTwoBefore()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "forty"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "one"));
+ SpanNotQuery query = new SpanNotQuery(near, term3, 2, 0);
+
+ CheckHits(query, new int[] { 840, 841, 842, 843, 844, 845, 846, 847, 848, 849 });
+
+ Assert.IsTrue(Searcher.Explain(query, 840).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 849).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanNotWindowNeg()
+ {
+ //test handling of invalid window < 0
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "eight"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "one"));
+ SpanNearQuery near = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 4, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "forty"));
+
+ SpanOrQuery or = new SpanOrQuery(term3);
+
+ SpanNotQuery query = new SpanNotQuery(near, or);
+
+ CheckHits(query, new int[] { 801, 821, 831, 851, 861, 871, 881, 891, 1801, 1821, 1831, 1851, 1861, 1871, 1881, 1891 });
+
+ Assert.IsTrue(Searcher.Explain(query, 801).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 891).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanNotWindowDoubleExcludesBefore()
+ {
+ //test hitting two excludes before an include
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "forty"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "two"));
+ SpanNearQuery near = new SpanNearQuery(new SpanTermQuery[] { term1, term2 }, 2, true);
+ SpanTermQuery exclude = new SpanTermQuery(new Term("field", "one"));
+
+ SpanNotQuery query = new SpanNotQuery(near, exclude, 4, 1);
+
+ CheckHits(query, new int[] { 42, 242, 342, 442, 542, 642, 742, 842, 942 });
+
+ Assert.IsTrue(Searcher.Explain(query, 242).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 942).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanFirst()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "five"));
+ SpanFirstQuery query = new SpanFirstQuery(term1, 1);
+
+ CheckHits(query, new int[] { 5, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599 });
+
+ Assert.IsTrue(Searcher.Explain(query, 5).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 599).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanPositionRange()
+ {
+ SpanPositionRangeQuery query;
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "five"));
+ query = new SpanPositionRangeQuery(term1, 1, 2);
+ CheckHits(query, new int[] { 25, 35, 45, 55, 65, 75, 85, 95 });
+ Assert.IsTrue(Searcher.Explain(query, 25).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 95).Value > 0.0f);
+
+ query = new SpanPositionRangeQuery(term1, 0, 1);
+ CheckHits(query, new int[] { 5, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599 });
+
+ query = new SpanPositionRangeQuery(term1, 6, 7);
+ CheckHits(query, new int[] { });
+ }
+
+ [Test]
+ public virtual void TestSpanPayloadCheck()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "five"));
+#pragma warning disable 612, 618
+ BytesRef pay = new BytesRef(("pos: " + 5).GetBytes(IOUtils.CHARSET_UTF_8));
+#pragma warning restore 612, 618
+ SpanQuery query = new SpanPayloadCheckQuery(term1, new List<byte[]>() { pay.Bytes });
+ CheckHits(query, new int[] { 1125, 1135, 1145, 1155, 1165, 1175, 1185, 1195, 1225, 1235, 1245, 1255, 1265, 1275, 1285, 1295, 1325, 1335, 1345, 1355, 1365, 1375, 1385, 1395, 1425, 1435, 1445, 1455, 1465, 1475, 1485, 1495, 1525, 1535, 1545, 1555, 1565, 1575, 1585, 1595, 1625, 1635, 1645, 1655, 1665, 1675, 1685, 1695, 1725, 1735, 1745, 1755, 1765, 1775, 1785, 1795, 1825, 1835, 1845, 1855, 1865, 1875, 1885, 1895, 1925, 1935, 1945, 1955, 1965, 1975, 1985, 1995 });
+ Assert.IsTrue(Searcher.Explain(query, 1125).Value > 0.0f);
+
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "hundred"));
+ SpanNearQuery snq;
+ SpanQuery[] clauses;
+ IList<byte[]> list;
+ BytesRef pay2;
+ clauses = new SpanQuery[2];
+ clauses[0] = term1;
+ clauses[1] = term2;
+ snq = new SpanNearQuery(clauses, 0, true);
+#pragma warning disable 612, 618
+ pay = new BytesRef(("pos: " + 0).GetBytes(IOUtils.CHARSET_UTF_8));
+ pay2 = new BytesRef(("pos: " + 1).GetBytes(IOUtils.CHARSET_UTF_8));
+#pragma warning restore 612, 618
+ list = new List<byte[]>();
+ list.Add(pay.Bytes);
+ list.Add(pay2.Bytes);
+ query = new SpanNearPayloadCheckQuery(snq, list);
+ CheckHits(query, new int[] { 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599 });
+ clauses = new SpanQuery[3];
+ clauses[0] = term1;
+ clauses[1] = term2;
+ clauses[2] = new SpanTermQuery(new Term("field", "five"));
+ snq = new SpanNearQuery(clauses, 0, true);
+#pragma warning disable 612, 618
+ pay = new BytesRef(("pos: " + 0).GetBytes(IOUtils.CHARSET_UTF_8));
+ pay2 = new BytesRef(("pos: " + 1).GetBytes(IOUtils.CHARSET_UTF_8));
+ BytesRef pay3 = new BytesRef(("pos: " + 2).GetBytes(IOUtils.CHARSET_UTF_8));
+#pragma warning restore 612, 618
+ list = new List<byte[]>();
+ list.Add(pay.Bytes);
+ list.Add(pay2.Bytes);
+ list.Add(pay3.Bytes);
+ query = new SpanNearPayloadCheckQuery(snq, list);
+ CheckHits(query, new int[] { 505 });
+ }
+
+ [Test]
+ public virtual void TestComplexSpanChecks()
+ {
+ SpanTermQuery one = new SpanTermQuery(new Term("field", "one"));
+ SpanTermQuery thous = new SpanTermQuery(new Term("field", "thousand"));
+ //should be one position in between
+ SpanTermQuery hundred = new SpanTermQuery(new Term("field", "hundred"));
+ SpanTermQuery three = new SpanTermQuery(new Term("field", "three"));
+
+ SpanNearQuery oneThous = new SpanNearQuery(new SpanQuery[] { one, thous }, 0, true);
+ SpanNearQuery hundredThree = new SpanNearQuery(new SpanQuery[] { hundred, three }, 0, true);
+ SpanNearQuery oneThousHunThree = new SpanNearQuery(new SpanQuery[] { oneThous, hundredThree }, 1, true);
+ SpanQuery query;
+ //this one's too small
+ query = new SpanPositionRangeQuery(oneThousHunThree, 1, 2);
+ CheckHits(query, new int[] { });
+ //this one's just right
+ query = new SpanPositionRangeQuery(oneThousHunThree, 0, 6);
+ CheckHits(query, new int[] { 1103, 1203, 1303, 1403, 1503, 1603, 1703, 1803, 1903 });
+
+ var payloads = new List<byte[]>();
+#pragma warning disable 612, 618
+ BytesRef pay = new BytesRef(("pos: " + 0).GetBytes(IOUtils.CHARSET_UTF_8));
+ BytesRef pay2 = new BytesRef(("pos: " + 1).GetBytes(IOUtils.CHARSET_UTF_8));
+ BytesRef pay3 = new BytesRef(("pos: " + 3).GetBytes(IOUtils.CHARSET_UTF_8));
+ BytesRef pay4 = new BytesRef(("pos: " + 4).GetBytes(IOUtils.CHARSET_UTF_8));
+#pragma warning restore 612, 618
+ payloads.Add(pay.Bytes);
+ payloads.Add(pay2.Bytes);
+ payloads.Add(pay3.Bytes);
+ payloads.Add(pay4.Bytes);
+ query = new SpanNearPayloadCheckQuery(oneThousHunThree, payloads);
+ CheckHits(query, new int[] { 1103, 1203, 1303, 1403, 1503, 1603, 1703, 1803, 1903 });
+ }
+
+ [Test]
+ public virtual void TestSpanOr()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "thirty"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "three"));
+ SpanNearQuery near1 = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 0, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "forty"));
+ SpanTermQuery term4 = new SpanTermQuery(new Term("field", "seven"));
+ SpanNearQuery near2 = new SpanNearQuery(new SpanQuery[] { term3, term4 }, 0, true);
+
+ SpanOrQuery query = new SpanOrQuery(near1, near2);
+
+ CheckHits(query, new int[] { 33, 47, 133, 147, 233, 247, 333, 347, 433, 447, 533, 547, 633, 647, 733, 747, 833, 847, 933, 947, 1033, 1047, 1133, 1147, 1233, 1247, 1333, 1347, 1433, 1447, 1533, 1547, 1633, 1647, 1733, 1747, 1833, 1847, 1933, 1947 });
+
+ Assert.IsTrue(Searcher.Explain(query, 33).Value > 0.0f);
+ Assert.IsTrue(Searcher.Explain(query, 947).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanExactNested()
+ {
+ SpanTermQuery term1 = new SpanTermQuery(new Term("field", "three"));
+ SpanTermQuery term2 = new SpanTermQuery(new Term("field", "hundred"));
+ SpanNearQuery near1 = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 0, true);
+ SpanTermQuery term3 = new SpanTermQuery(new Term("field", "thirty"));
+ SpanTermQuery term4 = new SpanTermQuery(new Term("field", "three"));
+ SpanNearQuery near2 = new SpanNearQuery(new SpanQuery[] { term3, term4 }, 0, true);
+
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { near1, near2 }, 0, true);
+
+ CheckHits(query, new int[] { 333, 1333 });
+
+ Assert.IsTrue(Searcher.Explain(query, 333).Value > 0.0f);
+ }
+
+ [Test]
+ public virtual void TestSpanNearOr()
+ {
+ SpanTermQuery t1 = new SpanTermQuery(new Term("field", "six"));
+ SpanTermQuery t3 = new SpanTermQuery(new Term("field", "seven"));
+
+ SpanTermQuery t5 = new SpanTermQuery(new Term("field", "seven"));
+ SpanTermQuery t6 = new SpanTermQuery(new Term("field", "six"));
+
+ SpanOrQuery to1 = new SpanOrQuery(t1, t3);
+ SpanOrQuery to2 = new SpanOrQuery(t5, t6);
+
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { to1, to2 }, 10, true);
+
+ CheckHits(query, new int[] { 606, 607, 626, 627, 636, 637, 646, 647, 656, 657, 666, 667, 676, 677, 686, 687, 696, 697, 706, 707, 726, 727, 736, 737, 746, 747, 756, 757, 766, 767, 776, 777, 786, 787, 796, 797, 1606, 1607, 1626, 1627, 1636, 1637, 1646, 1647, 1656, 1657, 1666, 1667, 1676, 1677, 1686, 1687, 1696, 1697, 1706, 1707, 1726, 1727, 1736, 1737, 1746, 1747, 1756, 1757, 1766, 1767, 1776, 1777, 1786, 1787, 1796, 1797 });
+ }
+
+ [Test]
+ public virtual void TestSpanComplex1()
+ {
+ SpanTermQuery t1 = new SpanTermQuery(new Term("field", "six"));
+ SpanTermQuery t2 = new SpanTermQuery(new Term("field", "hundred"));
+ SpanNearQuery tt1 = new SpanNearQuery(new SpanQuery[] { t1, t2 }, 0, true);
+
+ SpanTermQuery t3 = new SpanTermQuery(new Term("field", "seven"));
+ SpanTermQuery t4 = new SpanTermQuery(new Term("field", "hundred"));
+ SpanNearQuery tt2 = new SpanNearQuery(new SpanQuery[] { t3, t4 }, 0, true);
+
+ SpanTermQuery t5 = new SpanTermQuery(new Term("field", "seven"));
+ SpanTermQuery t6 = new SpanTermQuery(new Term("field", "six"));
+
+ SpanOrQuery to1 = new SpanOrQuery(tt1, tt2);
+ SpanOrQuery to2 = new SpanOrQuery(t5, t6);
+
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { to1, to2 }, 100, true);
+
+ CheckHits(query, new int[] { 606, 607, 626, 627, 636, 637, 646, 647, 656, 657, 666, 667, 676, 677, 686, 687, 696, 697, 706, 707, 726, 727, 736, 737, 746, 747, 756, 757, 766, 767, 776, 777, 786, 787, 796, 797, 1606, 1607, 1626, 1627, 1636, 1637, 1646, 1647, 1656, 1657, 1666, 1667, 1676, 1677, 1686, 1687, 1696, 1697, 1706, 1707, 1726, 1727, 1736, 1737, 1746, 1747, 1756, 1757, 1766, 1767, 1776, 1777, 1786, 1787, 1796, 1797 });
+ }
+
+ [Test]
+ public virtual void TestSpansSkipTo()
+ {
+ SpanTermQuery t1 = new SpanTermQuery(new Term("field", "seventy"));
+ SpanTermQuery t2 = new SpanTermQuery(new Term("field", "seventy"));
+ Spans s1 = MultiSpansWrapper.Wrap(Searcher.TopReaderContext, t1);
+ Spans s2 = MultiSpansWrapper.Wrap(Searcher.TopReaderContext, t2);
+
+ Assert.IsTrue(s1.Next());
+ Assert.IsTrue(s2.Next());
+
+ bool hasMore = true;
+
+ do
+ {
+ hasMore = SkipToAccoringToJavaDocs(s1, s1.Doc + 1);
+ Assert.AreEqual(hasMore, s2.SkipTo(s2.Doc + 1));
+ Assert.AreEqual(s1.Doc, s2.Doc);
+ } while (hasMore);
+ }
+
+ /// <summary>
+ /// Skips to the first match beyond the current, whose document number is
+ /// greater than or equal to <i>target</i>. <p>Returns true iff there is such
+ /// a match. <p>Behaves as if written: <pre>
+ /// boolean skipTo(int target) {
+ /// do {
+ /// if (!next())
+ /// return false;
+ /// } while (target > doc());
+ /// return true;
+ /// }
+ /// </pre>
+ /// </summary>
+ private bool SkipToAccoringToJavaDocs(Spans s, int target)
+ {
+ do
+ {
+ if (!s.Next())
+ {
+ return false;
+ }
+ } while (target > s.Doc);
+ return true;
+ }
+
+ private void CheckHits(Query query, int[] results)
+ {
+ Search.CheckHits.DoCheckHits(Random(), query, "field", Searcher, results, Similarity);
+ }
+ }
+}
\ No newline at end of file