You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/01/30 14:16:52 UTC
svn commit: r1562808 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/facet/ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/
lucene/facet/src/test/org/apache/lucene/facet/taxonomy/
Author: shaie
Date: Thu Jan 30 13:16:51 2014
New Revision: 1562808
URL: http://svn.apache.org/r1562808
Log:
LUCENE-5320: Add SearcherTaxonomyManager over Directory
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1562808&r1=1562807&r2=1562808&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Thu Jan 30 13:16:51 2014
@@ -63,6 +63,9 @@ New Features
* LUCENE-5353: ShingleFilter's filler token should be configurable.
(Ahmet Arslan, Simon Willnauer, Steve Rowe)
+* LUCENE-5320: Add SearcherTaxonomyManager over search and taxonomy index
+ directories (i.e. not only NRT). (Shai Erera)
+
Build
* LUCENE-5217,LUCENE-5420: Maven config: get dependencies from Ant+Ivy config;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java?rev=1562808&r1=1562807&r2=1562808&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java Thu Jan 30 13:16:51 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
/**
@@ -69,9 +70,27 @@ public class SearcherTaxonomyManager ext
this.searcherFactory = searcherFactory;
this.taxoWriter = taxoWriter;
DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
- current = new SearcherAndTaxonomy(SearcherManager.getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes)),
- taxoReader);
- taxoEpoch = taxoWriter.getTaxonomyEpoch();
+ current = new SearcherAndTaxonomy(SearcherManager.getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes)), taxoReader);
+ this.taxoEpoch = taxoWriter.getTaxonomyEpoch();
+ }
+
+ /**
+ * Creates search and taxonomy readers over the corresponding directories.
+ *
+ * <p>
+ * <b>NOTE:</b> you should only use this constructor if you commit and call
+ * {@link #maybeRefresh()} in the same thread. Otherwise it could lead to an
+ * unsync'd {@link IndexSearcher} and {@link TaxonomyReader} pair.
+ */
+ public SearcherTaxonomyManager(Directory indexDir, Directory taxoDir, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ current = new SearcherAndTaxonomy(SearcherManager.getSearcher(searcherFactory, DirectoryReader.open(indexDir)), taxoReader);
+ this.taxoWriter = null;
+ taxoEpoch = -1;
}
@Override
@@ -114,7 +133,7 @@ public class SearcherTaxonomyManager ext
if (tr == null) {
ref.taxonomyReader.incRef();
tr = ref.taxonomyReader;
- } else if (taxoWriter.getTaxonomyEpoch() != taxoEpoch) {
+ } else if (taxoWriter != null && taxoWriter.getTaxonomyEpoch() != taxoEpoch) {
IOUtils.close(newReader, tr);
throw new IllegalStateException("DirectoryTaxonomyWriter.replaceTaxonomy was called, which is not allowed when using SearcherTaxonomyManager");
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java?rev=1562808&r1=1562807&r2=1562808&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java Thu Jan 30 13:16:51 2014
@@ -37,12 +37,82 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util._TestUtil;
public class TestSearcherTaxonomyManager extends FacetTestCase {
- public void test() throws Exception {
+
+ private static class IndexerThread extends Thread {
+
+ private IndexWriter w;
+ private FacetsConfig config;
+ private TaxonomyWriter tw;
+ private ReferenceManager<SearcherAndTaxonomy> mgr;
+ private int ordLimit;
+ private AtomicBoolean stop;
+
+ public IndexerThread(IndexWriter w, FacetsConfig config, TaxonomyWriter tw,
+ ReferenceManager<SearcherAndTaxonomy> mgr, int ordLimit, AtomicBoolean stop) {
+ this.w = w;
+ this.config = config;
+ this.tw = tw;
+ this.mgr = mgr;
+ this.ordLimit = ordLimit;
+ this.stop = stop;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Set<String> seen = new HashSet<String>();
+ List<String> paths = new ArrayList<String>();
+ while (true) {
+ Document doc = new Document();
+ int numPaths = _TestUtil.nextInt(random(), 1, 5);
+ for(int i=0;i<numPaths;i++) {
+ String path;
+ if (!paths.isEmpty() && random().nextInt(5) != 4) {
+ // Use previous path
+ path = paths.get(random().nextInt(paths.size()));
+ } else {
+ // Create new path
+ path = null;
+ while (true) {
+ path = _TestUtil.randomRealisticUnicodeString(random());
+ if (path.length() != 0 && !seen.contains(path)) {
+ seen.add(path);
+ paths.add(path);
+ break;
+ }
+ }
+ }
+ doc.add(new FacetField("field", path));
+ }
+ try {
+ w.addDocument(config.build(tw, doc));
+ if (mgr != null && random().nextDouble() < 0.1) {
+ w.commit();
+ tw.commit();
+ mgr.maybeRefresh();
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ if (tw.getSize() >= ordLimit) {
+ break;
+ }
+ }
+ } finally {
+ stop.set(true);
+ }
+ }
+
+ }
+
+ public void testNRT() throws Exception {
Directory dir = newDirectory();
Directory taxoDir = newDirectory();
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -54,49 +124,7 @@ public class TestSearcherTaxonomyManager
// How many unique facets to index before stopping:
final int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
- Thread indexer = new Thread() {
- @Override
- public void run() {
- try {
- Set<String> seen = new HashSet<String>();
- List<String> paths = new ArrayList<String>();
- while (true) {
- Document doc = new Document();
- int numPaths = _TestUtil.nextInt(random(), 1, 5);
- for(int i=0;i<numPaths;i++) {
- String path;
- if (!paths.isEmpty() && random().nextInt(5) != 4) {
- // Use previous path
- path = paths.get(random().nextInt(paths.size()));
- } else {
- // Create new path
- path = null;
- while (true) {
- path = _TestUtil.randomRealisticUnicodeString(random());
- if (path.length() != 0 && !seen.contains(path)) {
- seen.add(path);
- paths.add(path);
- break;
- }
- }
- }
- doc.add(new FacetField("field", path));
- }
- try {
- w.addDocument(config.build(tw, doc));
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
-
- if (tw.getSize() >= ordLimit) {
- break;
- }
- }
- } finally {
- stop.set(true);
- }
- }
- };
+ Thread indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
final SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(w, true, null, tw);
@@ -160,8 +188,60 @@ public class TestSearcherTaxonomyManager
IOUtils.close(mgr, tw, w, taxoDir, dir);
}
+
+ public void testDirectory() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+ final IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+ // first empty commit
+ w.commit();
+ tw.commit();
+ final SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
+ final FacetsConfig config = new FacetsConfig();
+ config.setMultiValued("field", true);
+ final AtomicBoolean stop = new AtomicBoolean();
- public void testReplaceTaxonomy() throws Exception {
+ // How many unique facets to index before stopping:
+ final int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
+
+ Thread indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
+ indexer.start();
+
+ try {
+ while (!stop.get()) {
+ SearcherAndTaxonomy pair = mgr.acquire();
+ try {
+ //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
+ FacetsCollector sfc = new FacetsCollector();
+ pair.searcher.search(new MatchAllDocsQuery(), sfc);
+ Facets facets = getTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
+ FacetResult result = facets.getTopChildren(10, "field");
+ if (pair.searcher.getIndexReader().numDocs() > 0) {
+ //System.out.println(pair.taxonomyReader.getSize());
+ assertTrue(result.childCount > 0);
+ assertTrue(result.labelValues.length > 0);
+ }
+
+ //if (VERBOSE) {
+ //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
+ //}
+ } finally {
+ mgr.release(pair);
+ }
+ }
+ } finally {
+ indexer.join();
+ }
+
+ if (VERBOSE) {
+ System.out.println("TEST: now stop");
+ }
+
+ IOUtils.close(mgr, tw, w, taxoDir, indexDir);
+ }
+
+ public void testReplaceTaxonomyNRT() throws Exception {
Directory dir = newDirectory();
Directory taxoDir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -185,4 +265,43 @@ public class TestSearcherTaxonomyManager
IOUtils.close(mgr, tw, w, taxoDir, dir);
}
+
+ public void testReplaceTaxonomyDirectory() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+ IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+ w.commit();
+ tw.commit();
+
+ Directory taxoDir2 = newDirectory();
+ DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(taxoDir2);
+ tw2.addCategory(new FacetLabel("a", "b"));
+ tw2.close();
+
+ SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
+ SearcherAndTaxonomy pair = mgr.acquire();
+ try {
+ assertEquals(1, pair.taxonomyReader.getSize());
+ } finally {
+ mgr.release(pair);
+ }
+
+ w.addDocument(new Document());
+ tw.replaceTaxonomy(taxoDir2);
+ taxoDir2.close();
+ w.commit();
+ tw.commit();
+
+ mgr.maybeRefresh();
+ pair = mgr.acquire();
+ try {
+ assertEquals(3, pair.taxonomyReader.getSize());
+ } finally {
+ mgr.release(pair);
+ }
+
+ IOUtils.close(mgr, tw, w, taxoDir, indexDir);
+ }
+
}