You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2016/10/02 14:35:25 UTC
[07/50] [abbrv] lucenenet git commit: Completed port of
Suggest.Spell.TestSpellChecker.TestConcurrentAccess()
Completed port of Suggest.Spell.TestSpellChecker.TestConcurrentAccess()
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/c7ed8ccc
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/c7ed8ccc
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/c7ed8ccc
Branch: refs/heads/master
Commit: c7ed8ccc688a8049aae0a1e7f3ace36168a25e2d
Parents: e539eee
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Thu Sep 15 04:21:15 2016 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Sun Oct 2 17:44:06 2016 +0700
----------------------------------------------------------------------
.../Spell/TestSpellChecker.cs | 242 ++++++++++---------
1 file changed, 133 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c7ed8ccc/src/Lucene.Net.Tests.Suggest/Spell/TestSpellChecker.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Suggest/Spell/TestSpellChecker.cs b/src/Lucene.Net.Tests.Suggest/Spell/TestSpellChecker.cs
index 532c2a0..80701ef 100644
--- a/src/Lucene.Net.Tests.Suggest/Spell/TestSpellChecker.cs
+++ b/src/Lucene.Net.Tests.Suggest/Spell/TestSpellChecker.cs
@@ -2,11 +2,14 @@
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Store;
+using Lucene.Net.Support;
using Lucene.Net.Util;
using NUnit.Framework;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
+using System.Globalization;
+using System.Threading;
namespace Lucene.Net.Search.Spell
{
@@ -77,7 +80,6 @@ namespace Lucene.Net.Search.Spell
}
writer.Dispose();
- //searchers = Collections.SynchronizedList(new ArrayList<IndexSearcher>());
searchers = new ConcurrentBag<IndexSearcher>();
// create the spellChecker
spellindex = NewDirectory();
@@ -108,7 +110,7 @@ namespace Lucene.Net.Search.Spell
assertEquals(num_field2, num_field1 + 1);
- assertLastSearcherOpen(4);
+ AssertLastSearcherOpen(4);
CheckCommonSuggestions(r);
CheckLevenshteinSuggestions(r);
@@ -143,7 +145,7 @@ namespace Lucene.Net.Search.Spell
SpellChecker compareSP = new SpellCheckerMock(compIdx, new LevensteinDistance(), new SuggestWordFrequencyComparator());
addwords(r, compareSP, "field3");
- String[] similar = compareSP.SuggestSimilar("fvie", 2, r, "field3",
+ string[] similar = compareSP.SuggestSimilar("fvie", 2, r, "field3",
SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
assertTrue(similar.Length == 2);
//five and fvei have the same score, but different frequencies.
@@ -181,7 +183,7 @@ namespace Lucene.Net.Search.Spell
{
- String[] similar = spellChecker.SuggestSimilar("eighty", 2, r, "field1",
+ string[] similar = spellChecker.SuggestSimilar("eighty", 2, r, "field1",
SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
assertEquals(1, similar.Length);
assertEquals("eighty", similar[0]);
@@ -189,7 +191,7 @@ namespace Lucene.Net.Search.Spell
{
- String[] similar = spellChecker.SuggestSimilar("eight", 2, r, "field1",
+ string[] similar = spellChecker.SuggestSimilar("eight", 2, r, "field1",
SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
assertEquals(1, similar.Length);
assertEquals("eight", similar[0]);
@@ -197,7 +199,7 @@ namespace Lucene.Net.Search.Spell
{
- String[] similar = spellChecker.SuggestSimilar("eighty", 5, r, "field1",
+ string[] similar = spellChecker.SuggestSimilar("eighty", 5, r, "field1",
SuggestMode.SUGGEST_MORE_POPULAR);
assertEquals(5, similar.Length);
assertEquals("eight", similar[0]);
@@ -205,7 +207,7 @@ namespace Lucene.Net.Search.Spell
{
- String[] similar = spellChecker.SuggestSimilar("twenty", 5, r, "field1",
+ string[] similar = spellChecker.SuggestSimilar("twenty", 5, r, "field1",
SuggestMode.SUGGEST_MORE_POPULAR);
assertEquals(1, similar.Length);
assertEquals("twenty-one", similar[0]);
@@ -213,14 +215,14 @@ namespace Lucene.Net.Search.Spell
{
- String[] similar = spellChecker.SuggestSimilar("eight", 5, r, "field1",
+ string[] similar = spellChecker.SuggestSimilar("eight", 5, r, "field1",
SuggestMode.SUGGEST_MORE_POPULAR);
assertEquals(0, similar.Length);
}
{
- String[] similar = spellChecker.SuggestSimilar("eighty", 5, r, "field1",
+ string[] similar = spellChecker.SuggestSimilar("eighty", 5, r, "field1",
SuggestMode.SUGGEST_ALWAYS);
assertEquals(5, similar.Length);
assertEquals("eight", similar[0]);
@@ -376,7 +378,7 @@ namespace Lucene.Net.Search.Spell
int num_field2 = this.NumDoc();
assertEquals(num_field2, num_field1 + 1);
CheckCommonSuggestions(r);
- assertLastSearcherOpen(4);
+ AssertLastSearcherOpen(4);
spellChecker.Dispose();
AssertSearchersClosed();
try
@@ -432,61 +434,68 @@ namespace Lucene.Net.Search.Spell
r.Dispose();
}
- // /*
- // * tests if the internally shared indexsearcher is correctly closed
- // * when the spellchecker is concurrently accessed and closed.
- // */
- // // LUCENENET TODO: Fisish port
- // [Test]
- // public void TestConcurrentAccess() {
- // assertEquals(1, searchers.Count);
- // IndexReader r = DirectoryReader.Open(userindex);
- // spellChecker.ClearIndex();
- // assertEquals(2, searchers.Count);
- // addwords(r, spellChecker, "field1");
- // assertEquals(3, searchers.Count);
- // int num_field1 = this.numdoc();
- // addwords(r, spellChecker, "field2");
- // assertEquals(4, searchers.Count);
- // int num_field2 = this.numdoc();
- // assertEquals(num_field2, num_field1 + 1);
- //int numThreads = 5 + Random().nextInt(5);
- //ExecutorService executor = Executors.newFixedThreadPool(numThreads, new NamedThreadFactory("testConcurrentAccess"));
- //SpellCheckWorker[] workers = new SpellCheckWorker[numThreads];
- // for (int i = 0; i<numThreads; i++) {
- // SpellCheckWorker spellCheckWorker = new SpellCheckWorker(r);
- //executor.execute(spellCheckWorker);
- // workers[i] = spellCheckWorker;
-
- // }
- // int iterations = 5 + Random().nextInt(5);
- // for (int i = 0; i<iterations; i++) {
- // Thread.Sleep(100);
- // // concurrently reset the spell index
- // spellChecker.SpellIndex=(this.spellindex);
- //// for debug - prints the internal open searchers
- //// showSearchersOpen();
- //}
-
- // spellChecker.Dispose();
- // executor.shutdown();
- // // wait for 60 seconds - usually this is very fast but coverage runs could take quite long
- // executor.awaitTermination(60L, TimeUnit.SECONDS);
-
- // for (int i = 0; i<workers.Length; i++) {
- // assertFalse(string.Format(CultureInfo.InvariantCulture, "worker thread {0} failed", i), workers[i].failed);
- // assertTrue(string.Format(CultureInfo.InvariantCulture, "worker thread {0} is still running but should be terminated", i), workers[i].terminated);
- // }
- // // 4 searchers more than iterations
- // // 1. at creation
- // // 2. clearIndex()
- // // 2. and 3. during addwords
- // assertEquals(iterations + 4, searchers.Count);
- // assertSearchersClosed();
- // r.Dispose();
- // }
+ /*
+ * tests if the internally shared indexsearcher is correctly closed
+ * when the spellchecker is concurrently accessed and closed.
+ */
+ [Test]
+ public void TestConcurrentAccess()
+ {
+ assertEquals(1, searchers.Count);
+ IndexReader r = DirectoryReader.Open(userindex);
+ spellChecker.ClearIndex();
+ assertEquals(2, searchers.Count);
+ addwords(r, spellChecker, "field1");
+ assertEquals(3, searchers.Count);
+ int num_field1 = this.NumDoc();
+ addwords(r, spellChecker, "field2");
+ assertEquals(4, searchers.Count);
+ int num_field2 = this.NumDoc();
+ assertEquals(num_field2, num_field1 + 1);
+ int numThreads = 5 + Random().nextInt(5);
+ SpellCheckWorker[] workers = new SpellCheckWorker[numThreads];
+ var stop = new AtomicBoolean(false);
+ for (int i = 0; i < numThreads; i++)
+ {
+ SpellCheckWorker spellCheckWorker = new SpellCheckWorker(this, r, stop);
+ workers[i] = spellCheckWorker;
+ spellCheckWorker.Start();
+ }
+ int iterations = 5 + Random().nextInt(5);
+ for (int i = 0; i < iterations; i++)
+ {
+ Thread.Sleep(100);
+ // concurrently reset the spell index
+ spellChecker.SpellIndex = (this.spellindex);
+ // for debug - prints the internal open searchers
+ // showSearchersOpen();
+ }
+
+ spellChecker.Dispose();
+ stop.Set(true);
- private void assertLastSearcherOpen(int numSearchers)
+ // wait for 60 seconds - usually this is very fast but coverage runs could take quite long
+ //executor.awaitTermination(60L, TimeUnit.SECONDS);
+ foreach (SpellCheckWorker worker in workers)
+ {
+ worker.Join((long)TimeSpan.FromSeconds(60).TotalMilliseconds);
+ }
+
+ for (int i = 0; i < workers.Length; i++)
+ {
+ assertFalse(string.Format(CultureInfo.InvariantCulture, "worker thread {0} failed \n" + workers[i].Error.ToString(), i), workers[i].Error == null);
+ assertTrue(string.Format(CultureInfo.InvariantCulture, "worker thread {0} is still running but should be terminated", i), workers[i].terminated);
+ }
+ // 4 searchers more than iterations
+ // 1. at creation
+ // 2. clearIndex()
+ // 2. and 3. during addwords
+ assertEquals(iterations + 4, searchers.Count);
+ AssertSearchersClosed();
+ r.Dispose();
+ }
+
+ private void AssertLastSearcherOpen(int numSearchers)
{
assertEquals(numSearchers, searchers.Count);
IndexSearcher[] searcherArray = searchers.ToArray();
@@ -523,53 +532,68 @@ namespace Lucene.Net.Search.Spell
// System.out.println(count);
// }
+ private class SpellCheckWorker : ThreadClass
+ {
+ private readonly TestSpellChecker outerInstance;
- // LUCENENET TODO: Fisish port
- //private class SpellCheckWorker implements Runnable
- //{
- // private final IndexReader reader;
- // volatile boolean terminated = false;
- //volatile boolean failed = false;
-
-
- // SpellCheckWorker(IndexReader reader)
- //{
- // super();
- // this.reader = reader;
- //}
-
- //@Override
- // public void run()
- //{
- // try
- // {
- // while (true)
- // {
- // try
- // {
- // checkCommonSuggestions(reader);
- // }
- // catch (AlreadyClosedException e)
- // {
-
- // return;
- // }
- // catch (Throwable e)
- // {
-
- // e.printStackTrace();
- // failed = true;
- // return;
- // }
- // }
- // }
- // finally
- // {
- // terminated = true;
- // }
- //}
+ private readonly IndexReader reader;
+ private readonly AtomicBoolean stop;
+ private volatile Exception error;
+ internal volatile bool terminated = false;
- // }
+ public SpellCheckWorker(TestSpellChecker outerInstance, IndexReader reader, AtomicBoolean stop)
+ {
+ this.outerInstance = outerInstance;
+ this.reader = reader;
+ this.stop = stop;
+ }
+
+ public Exception Error
+ {
+ get { return error; }
+ }
+
+ public override void Run()
+ {
+ Priority += 1;
+ try
+ {
+ while (!stop.Get())
+ {
+ try
+ {
+ outerInstance.CheckCommonSuggestions(reader);
+
+ Thread.Sleep(10);// don't starve refresh()'s CPU, which sleeps every 50 bytes for 1 ms
+ }
+ catch (Exception e)
+ {
+ error = e;
+ stop.Set(true);
+ }
+ }
+ }
+ finally
+ {
+ terminated = true;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Grab the stack trace into a string since the exception was thrown in a thread and we want the assert
+ /// outside the thread to show the stack trace in case of failure.
+ /// </summary>
+ private string stackTraceStr(Exception error)
+ {
+ if (error == null)
+ {
+ return "";
+ }
+
+ error.printStackTrace();
+ return error.StackTrace;
+ }
internal class SpellCheckerMock : SpellChecker
{