You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2016/12/29 21:13:01 UTC
[1/4] lucene-solr:master: LUCENE-7564: Force single-threaded access
to the AnalyzingInfixSuggester's SearcherManager when performing an acquire()
or reassigning. This fixes failures in
AnalyzingInfixSuggester.testRandomNRT().
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 0d830a765 -> 266ca2640
refs/heads/master a4335c0e9 -> 6b00ee517
LUCENE-7564: Force single-threaded access to the AnalyzingInfixSuggester's SearcherManager when performing an acquire() or reassigning. This fixes failures in AnalyzingInfixSuggester.testRandomNRT().
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/93fdc207
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/93fdc207
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/93fdc207
Branch: refs/heads/master
Commit: 93fdc20736d6e13736aceb091ab978bd8e03fcbb
Parents: a4335c0
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Dec 29 15:51:37 2016 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Dec 29 15:51:37 2016 -0500
----------------------------------------------------------------------
.../analyzing/AnalyzingInfixSuggester.java | 136 +++++++++++--------
1 file changed, 81 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93fdc207/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
index b8c2dbd..2fbe4a8 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
@@ -136,6 +136,8 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
/** {@link IndexSearcher} used for lookups. */
protected SearcherManager searcherMgr;
+
+ protected final Object searcherMgrLock = new Object();
/** Default minimum number of leading characters before
* PrefixQuery is used (4). */
@@ -275,53 +277,55 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
@Override
public void build(InputIterator iter) throws IOException {
- if (searcherMgr != null) {
- searcherMgr.close();
- searcherMgr = null;
- }
+ synchronized (searcherMgrLock) {
+ if (searcherMgr != null) {
+ searcherMgr.close();
+ searcherMgr = null;
+ }
- if (writer != null) {
- writer.close();
- writer = null;
- }
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
- boolean success = false;
- try {
- // First pass: build a temporary normal Lucene index,
- // just indexing the suggestions as they iterate:
- writer = new IndexWriter(dir,
- getIndexWriterConfig(getGramAnalyzer(), IndexWriterConfig.OpenMode.CREATE));
- //long t0 = System.nanoTime();
-
- // TODO: use threads?
- BytesRef text;
- while ((text = iter.next()) != null) {
- BytesRef payload;
- if (iter.hasPayloads()) {
- payload = iter.payload();
- } else {
- payload = null;
- }
+ boolean success = false;
+ try {
+ // First pass: build a temporary normal Lucene index,
+ // just indexing the suggestions as they iterate:
+ writer = new IndexWriter(dir,
+ getIndexWriterConfig(getGramAnalyzer(), IndexWriterConfig.OpenMode.CREATE));
+ //long t0 = System.nanoTime();
+
+ // TODO: use threads?
+ BytesRef text;
+ while ((text = iter.next()) != null) {
+ BytesRef payload;
+ if (iter.hasPayloads()) {
+ payload = iter.payload();
+ } else {
+ payload = null;
+ }
- add(text, iter.contexts(), iter.weight(), payload);
- }
+ add(text, iter.contexts(), iter.weight(), payload);
+ }
- //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");
- if (commitOnBuild || closeIndexWriterOnBuild) {
- commit();
- }
- searcherMgr = new SearcherManager(writer, null);
- success = true;
- } finally {
- if (success) {
- if (closeIndexWriterOnBuild) {
- writer.close();
- writer = null;
+ //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");
+ if (commitOnBuild || closeIndexWriterOnBuild) {
+ commit();
}
- } else { // failure
- if (writer != null) {
- writer.rollback();
- writer = null;
+ searcherMgr = new SearcherManager(writer, null);
+ success = true;
+ } finally {
+ if (success) {
+ if (closeIndexWriterOnBuild) {
+ writer.close();
+ writer = null;
+ }
+ } else { // failure
+ if (writer != null) {
+ writer.rollback();
+ writer = null;
+ }
}
}
}
@@ -369,10 +373,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
} else {
writer = new IndexWriter(dir, getIndexWriterConfig(getGramAnalyzer(), IndexWriterConfig.OpenMode.CREATE));
}
- SearcherManager oldSearcherMgr = searcherMgr;
- searcherMgr = new SearcherManager(writer, null);
- if (oldSearcherMgr != null) {
- oldSearcherMgr.close();
+ synchronized (searcherMgrLock) {
+ SearcherManager oldSearcherMgr = searcherMgr;
+ searcherMgr = new SearcherManager(writer, null);
+ if (oldSearcherMgr != null) {
+ oldSearcherMgr.close();
+ }
}
}
}
@@ -642,7 +648,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
// only retrieve the first num hits now:
Collector c2 = new EarlyTerminatingSortingCollector(c, SORT, num);
List<LookupResult> results = null;
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
//System.out.println("got searcher=" + searcher);
searcher.search(finalQuery, c2);
@@ -653,7 +664,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
// hits = searcher.search(query, null, num, SORT);
results = createResults(searcher, hits, num, key, doHighlight, matchedTokens, prefixToken);
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
//System.out.println((System.currentTimeMillis() - t0) + " msec for infix suggest");
@@ -853,7 +864,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
long mem = RamUsageEstimator.shallowSizeOf(this);
try {
if (searcherMgr != null) {
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
LeafReader reader = FilterLeafReader.unwrap(context.reader());
@@ -862,7 +878,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
}
}
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
}
return mem;
@@ -876,7 +892,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
List<Accountable> resources = new ArrayList<>();
try {
if (searcherMgr != null) {
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
LeafReader reader = FilterLeafReader.unwrap(context.reader());
@@ -885,7 +906,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
}
}
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
}
return Collections.unmodifiableList(resources);
@@ -899,11 +920,16 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
if (searcherMgr == null) {
return 0;
}
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
return searcher.getIndexReader().numDocs();
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
}
-};
+}
[4/4] lucene-solr:master: LUCENE-7564: add missing javadocs
Posted by sa...@apache.org.
LUCENE-7564: add missing javadocs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6b00ee51
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6b00ee51
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6b00ee51
Branch: refs/heads/master
Commit: 6b00ee5175d55d2f2a25ce6539dc12277022c898
Parents: 93fdc20
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Dec 29 16:08:35 2016 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Dec 29 16:08:53 2016 -0500
----------------------------------------------------------------------
.../lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6b00ee51/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
index 2fbe4a8..81880d4 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
@@ -137,6 +137,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
/** {@link IndexSearcher} used for lookups. */
protected SearcherManager searcherMgr;
+ /** Used to manage concurrent access to searcherMgr */
protected final Object searcherMgrLock = new Object();
/** Default minimum number of leading characters before
[3/4] lucene-solr:branch_6x: LUCENE-7564: add missing javadocs
Posted by sa...@apache.org.
LUCENE-7564: add missing javadocs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/266ca264
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/266ca264
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/266ca264
Branch: refs/heads/branch_6x
Commit: 266ca264077671f40a381c4768c8c6a86275b268
Parents: 73f068e
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Dec 29 16:08:35 2016 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Dec 29 16:08:35 2016 -0500
----------------------------------------------------------------------
.../lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/266ca264/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
index 0f1179f..eb6c885 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
@@ -137,6 +137,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
/** {@link IndexSearcher} used for lookups. */
protected SearcherManager searcherMgr;
+ /** Used to manage concurrent access to searcherMgr */
protected final Object searcherMgrLock = new Object();
/** Default minimum number of leading characters before
[2/4] lucene-solr:branch_6x: LUCENE-7564: Force single-threaded
access to the AnalyzingInfixSuggester's SearcherManager when performing an
acquire() or reassigning. This fixes failures in
AnalyzingInfixSuggester.testRandomNRT().
Posted by sa...@apache.org.
LUCENE-7564: Force single-threaded access to the AnalyzingInfixSuggester's SearcherManager when performing an acquire() or reassigning. This fixes failures in AnalyzingInfixSuggester.testRandomNRT().
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/73f068e5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/73f068e5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/73f068e5
Branch: refs/heads/branch_6x
Commit: 73f068e50333902b3ea887100f063e61ebf1996b
Parents: 0d830a7
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Dec 29 15:51:37 2016 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Dec 29 15:52:20 2016 -0500
----------------------------------------------------------------------
.../analyzing/AnalyzingInfixSuggester.java | 136 +++++++++++--------
1 file changed, 81 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/73f068e5/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
index 1e98361..0f1179f 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
@@ -136,6 +136,8 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
/** {@link IndexSearcher} used for lookups. */
protected SearcherManager searcherMgr;
+
+ protected final Object searcherMgrLock = new Object();
/** Default minimum number of leading characters before
* PrefixQuery is used (4). */
@@ -275,53 +277,55 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
@Override
public void build(InputIterator iter) throws IOException {
- if (searcherMgr != null) {
- searcherMgr.close();
- searcherMgr = null;
- }
+ synchronized (searcherMgrLock) {
+ if (searcherMgr != null) {
+ searcherMgr.close();
+ searcherMgr = null;
+ }
- if (writer != null) {
- writer.close();
- writer = null;
- }
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
- boolean success = false;
- try {
- // First pass: build a temporary normal Lucene index,
- // just indexing the suggestions as they iterate:
- writer = new IndexWriter(dir,
- getIndexWriterConfig(getGramAnalyzer(), IndexWriterConfig.OpenMode.CREATE));
- //long t0 = System.nanoTime();
-
- // TODO: use threads?
- BytesRef text;
- while ((text = iter.next()) != null) {
- BytesRef payload;
- if (iter.hasPayloads()) {
- payload = iter.payload();
- } else {
- payload = null;
- }
+ boolean success = false;
+ try {
+ // First pass: build a temporary normal Lucene index,
+ // just indexing the suggestions as they iterate:
+ writer = new IndexWriter(dir,
+ getIndexWriterConfig(getGramAnalyzer(), IndexWriterConfig.OpenMode.CREATE));
+ //long t0 = System.nanoTime();
+
+ // TODO: use threads?
+ BytesRef text;
+ while ((text = iter.next()) != null) {
+ BytesRef payload;
+ if (iter.hasPayloads()) {
+ payload = iter.payload();
+ } else {
+ payload = null;
+ }
- add(text, iter.contexts(), iter.weight(), payload);
- }
+ add(text, iter.contexts(), iter.weight(), payload);
+ }
- //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");
- if (commitOnBuild || closeIndexWriterOnBuild) {
- commit();
- }
- searcherMgr = new SearcherManager(writer, null);
- success = true;
- } finally {
- if (success) {
- if (closeIndexWriterOnBuild) {
- writer.close();
- writer = null;
+ //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");
+ if (commitOnBuild || closeIndexWriterOnBuild) {
+ commit();
}
- } else { // failure
- if (writer != null) {
- writer.rollback();
- writer = null;
+ searcherMgr = new SearcherManager(writer, null);
+ success = true;
+ } finally {
+ if (success) {
+ if (closeIndexWriterOnBuild) {
+ writer.close();
+ writer = null;
+ }
+ } else { // failure
+ if (writer != null) {
+ writer.rollback();
+ writer = null;
+ }
}
}
}
@@ -369,10 +373,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
} else {
writer = new IndexWriter(dir, getIndexWriterConfig(getGramAnalyzer(), IndexWriterConfig.OpenMode.CREATE));
}
- SearcherManager oldSearcherMgr = searcherMgr;
- searcherMgr = new SearcherManager(writer, null);
- if (oldSearcherMgr != null) {
- oldSearcherMgr.close();
+ synchronized (searcherMgrLock) {
+ SearcherManager oldSearcherMgr = searcherMgr;
+ searcherMgr = new SearcherManager(writer, null);
+ if (oldSearcherMgr != null) {
+ oldSearcherMgr.close();
+ }
}
}
}
@@ -642,7 +648,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
// only retrieve the first num hits now:
Collector c2 = new EarlyTerminatingSortingCollector(c, SORT, num);
List<LookupResult> results = null;
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
//System.out.println("got searcher=" + searcher);
searcher.search(finalQuery, c2);
@@ -653,7 +664,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
// hits = searcher.search(query, null, num, SORT);
results = createResults(searcher, hits, num, key, doHighlight, matchedTokens, prefixToken);
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
//System.out.println((System.currentTimeMillis() - t0) + " msec for infix suggest");
@@ -846,7 +857,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
long mem = RamUsageEstimator.shallowSizeOf(this);
try {
if (searcherMgr != null) {
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
LeafReader reader = FilterLeafReader.unwrap(context.reader());
@@ -855,7 +871,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
}
}
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
}
return mem;
@@ -869,7 +885,12 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
List<Accountable> resources = new ArrayList<>();
try {
if (searcherMgr != null) {
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
LeafReader reader = FilterLeafReader.unwrap(context.reader());
@@ -878,7 +899,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
}
}
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
}
return Collections.unmodifiableList(resources);
@@ -892,11 +913,16 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
if (searcherMgr == null) {
return 0;
}
- IndexSearcher searcher = searcherMgr.acquire();
+ SearcherManager mgr;
+ IndexSearcher searcher;
+ synchronized (searcherMgrLock) {
+ mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
+ searcher = mgr.acquire();
+ }
try {
return searcher.getIndexReader().numDocs();
} finally {
- searcherMgr.release(searcher);
+ mgr.release(searcher);
}
}
-};
+}