You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2012/05/16 09:32:54 UTC
svn commit: r1339032 - in
/lucene/dev/branches/lucene_solr_3_6/lucene/contrib: ./
facet/src/examples/org/apache/lucene/facet/example/merge/
facet/src/java/org/apache/lucene/facet/taxonomy/directory/
facet/src/test/org/apache/lucene/facet/taxonomy/direc...
Author: shaie
Date: Wed May 16 07:32:54 2012
New Revision: 1339032
URL: http://svn.apache.org/viewvc?rev=1339032&view=rev
Log:
LUCENE-4060: Fix DirectoryTaxonomyWriter.addTaxonomy to work in parallel with addCategory
Added:
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java
- copied, changed from r1339027, lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomies.java
Removed:
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomies.java
Modified:
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt?rev=1339032&r1=1339031&r2=1339032&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt Wed May 16 07:32:54 2012
@@ -25,6 +25,11 @@ Bug Fixes
wrongly used minimum sort buffer resulting in potentially slow FST &
Dictionary building. (Simon Willnauer)
+ * LUCENE-4060: Fix a synchronization bug in
+ DirectoryTaxonomyWriter.addTaxonomies(). Also, the method has been renamed to
+ addTaxonomy and now takes only one Directory and one OrdinalMap.
+ (Shai Erera, Gilad Barkai)
+
Tests
* LUCENE-3919, LUCENE-3969, LUCENE-3990: more thorough testing of analysis
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java?rev=1339032&r1=1339031&r2=1339032&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java Wed May 16 07:32:54 2012
@@ -81,7 +81,7 @@ public class TaxonomyMergeUtils {
OrdinalMap map, IndexWriter destIndexWriter,
DirectoryTaxonomyWriter destTaxWriter) throws IOException {
// merge the taxonomies
- destTaxWriter.addTaxonomies(new Directory[] { srcTaxDir }, new OrdinalMap[] { map });
+ destTaxWriter.addTaxonomy(srcTaxDir, map);
PayloadProcessorProvider payloadProcessor = new FacetsPayloadProcessorProvider(
srcIndexDir, map.getMap(), new DefaultFacetIndexingParams());
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java?rev=1339032&r1=1339031&r2=1339032&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java Wed May 16 07:32:54 2012
@@ -2,6 +2,7 @@ package org.apache.lucene.facet.taxonomy
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.index.Term;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -26,6 +27,12 @@ import org.apache.lucene.document.FieldS
abstract class Consts {
static final String FULL = "$full_path$";
+ /**
+ * A {@link #FULL} {@link Term} - use it for creating new terms instead of
+ * calling {@link Term#Term(String, String)}, by calling
+ * {@link Term#createTerm(String)} -- the latter does not do string interning.
+ */
+ static final Term FULL_TERM = new Term(FULL);
static final String FIELD_PAYLOADS = "$payloads$";
static final String PAYLOAD_PARENT = "p";
static final char[] PAYLOAD_PARENT_CHARS = PAYLOAD_PARENT.toCharArray();
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java?rev=1339032&r1=1339031&r2=1339032&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java Wed May 16 07:32:54 2012
@@ -189,7 +189,7 @@ public class DirectoryTaxonomyReader imp
int ret = TaxonomyReader.INVALID_ORDINAL;
try {
indexReaderLock.readLock().lock();
- TermDocs docs = indexReader.termDocs(new Term(Consts.FULL, path));
+ TermDocs docs = indexReader.termDocs(Consts.FULL_TERM.createTerm(path));
if (docs.next()) {
ret = docs.doc();
}
@@ -271,8 +271,7 @@ public class DirectoryTaxonomyReader imp
if (catID<0 || catID>=indexReader.maxDoc()) {
return null;
}
- ret = indexReader.document(catID, Consts.fullPathSelector)
- .get(Consts.FULL);
+ ret = indexReader.document(catID, Consts.fullPathSelector).get(Consts.FULL);
} finally {
indexReaderLock.readLock().unlock();
}
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1339032&r1=1339031&r2=1339032&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Wed May 16 07:32:54 2012
@@ -411,8 +411,7 @@ public class DirectoryTaxonomyWriter imp
reader = openReader();
}
- TermDocs docs = reader.termDocs(new Term(Consts.FULL, categoryPath
- .toString(delimiter)));
+ TermDocs docs = reader.termDocs(Consts.FULL_TERM.createTerm(categoryPath.toString(delimiter)));
if (!docs.next()) {
return -1; // category does not exist in taxonomy
}
@@ -446,7 +445,7 @@ public class DirectoryTaxonomyWriter imp
if (reader == null) {
reader = openReader();
}
- TermDocs docs = reader.termDocs(new Term(Consts.FULL, categoryPath
+ TermDocs docs = reader.termDocs(Consts.FULL_TERM.createTerm(categoryPath
.toString(delimiter, prefixLen)));
if (!docs.next()) {
return -1; // category does not exist in taxonomy
@@ -754,9 +753,8 @@ public class DirectoryTaxonomyWriter imp
CategoryPath cp = new CategoryPath();
TermDocs td = reader.termDocs();
- Term fullPathTerm = new Term(Consts.FULL);
- String field = fullPathTerm.field(); // needed so we can later use !=
- TermEnum terms = reader.terms(fullPathTerm);
+ String field = Consts.FULL_TERM.field(); // needed so we can later use !=
+ TermEnum terms = reader.terms(Consts.FULL_TERM);
// The check is done here to avoid checking it on every iteration of the
// below loop. A null term wlil be returned if there are no terms in the
// lexicon, or after the Consts.FULL term. However while the loop is
@@ -798,6 +796,7 @@ public class DirectoryTaxonomyWriter imp
}
return parentArray;
}
+
public int getParent(int ordinal) throws IOException {
ensureOpen();
// Note: the following if() just enforces that a user can never ask
@@ -808,154 +807,50 @@ public class DirectoryTaxonomyWriter imp
}
return getParentArray().getArray()[ordinal];
}
-
+
/**
- * Take all the categories of one or more given taxonomies, and add them to
- * the main taxonomy (this), if they are not already there.
- * <P>
- * Additionally, fill a <I>mapping</I> for each of the added taxonomies,
- * mapping its ordinals to the ordinals in the enlarged main taxonomy.
- * These mapping are saved into an array of OrdinalMap objects given by the
- * user, one for each of the given taxonomies (not including "this", the main
- * taxonomy). Often the first of these will be a MemoryOrdinalMap and the
- * others will be a DiskOrdinalMap - see discussion in {OrdinalMap}.
- * <P>
- * Note that the taxonomies to be added are given as Directory objects,
- * not opened TaxonomyReader/TaxonomyWriter objects, so if any of them are
- * currently managed by an open TaxonomyWriter, make sure to commit() (or
- * close()) it first. The main taxonomy (this) is an open TaxonomyWriter,
- * and does not need to be commit()ed before this call.
+ * Takes the categories from the given taxonomy directory, and adds the
+ * missing ones to this taxonomy. Additionally, it fills the given
+ * {@link OrdinalMap} with a mapping from the original ordinal to the new
+ * ordinal.
*/
- public void addTaxonomies(Directory[] taxonomies, OrdinalMap[] ordinalMaps) throws IOException {
+ public void addTaxonomy(Directory taxoDir, OrdinalMap map) throws IOException {
ensureOpen();
- // To prevent us stepping on the rest of this class's decisions on when
- // to open a reader, and when not, we'll be opening a new reader instead
- // of using the existing "reader" object:
- IndexReader mainreader = openReader();
- TermEnum mainte = mainreader.terms(new Term(Consts.FULL));
-
- IndexReader[] otherreaders = new IndexReader[taxonomies.length];
- TermEnum[] othertes = new TermEnum[taxonomies.length];
- for (int i=0; i<taxonomies.length; i++) {
- otherreaders[i] = IndexReader.open(taxonomies[i]);
- othertes[i] = otherreaders[i].terms(new Term(Consts.FULL));
- // Also tell the ordinal maps their expected sizes:
- ordinalMaps[i].setSize(otherreaders[i].numDocs());
- }
-
- CategoryPath cp = new CategoryPath();
-
- // We keep a "current" cursor over the alphabetically-ordered list of
- // categories in each taxonomy. We start the cursor on the first
- // (alphabetically) category of each taxonomy:
-
- String currentMain;
- String[] currentOthers = new String[taxonomies.length];
- currentMain = nextTE(mainte);
- int otherTaxonomiesLeft = 0;
- for (int i=0; i<taxonomies.length; i++) {
- currentOthers[i] = nextTE(othertes[i]);
- if (currentOthers[i]!=null) {
- otherTaxonomiesLeft++;
- }
- }
-
- // And then, at each step look at the first (alphabetically) of the
- // current taxonomies.
- // NOTE: The most efficient way we could have done this is using a
- // PriorityQueue. But for simplicity, and assuming that usually we'll
- // have a very small number of other taxonomies (often just 1), we use
- // a more naive algorithm (o(ntaxonomies) instead of o(ln ntaxonomies)
- // per step)
-
- while (otherTaxonomiesLeft>0) {
- String first=null;
- for (int i=0; i<taxonomies.length; i++) {
- if (currentOthers[i]==null) continue;
- if (first==null || first.compareTo(currentOthers[i])>0) {
- first = currentOthers[i];
- }
- }
- int comp = 0;
- if (currentMain==null || (comp = currentMain.compareTo(first))>0) {
- // If 'first' is before currentMain, or currentMain is null,
- // then 'first' is a new category and we need to add it to the
- // main taxonomy. Then for all taxonomies with this 'first'
- // category, we need to add the new category number to their
- // map, and move to the next category in all of them.
+ IndexReader r = IndexReader.open(taxoDir);
+ try {
+ final int size = r.numDocs();
+ final OrdinalMap ordinalMap = map;
+ ordinalMap.setSize(size);
+ CategoryPath cp = new CategoryPath();
+ TermEnum te = r.terms(Consts.FULL_TERM);
+ TermDocs docs = r.termDocs();
+ // we call next() first, to skip the root category which always exists.
+ while (te.next()) {
+ Term term = te.term();
+ if (term.field() != Consts.FULL) break;
cp.clear();
- cp.add(first, delimiter);
- // We can call internalAddCategory() instead of addCategory()
- // because we know the category hasn't been seen yet.
- int newordinal = internalAddCategory(cp, cp.length());
- // TODO (Facet): we already had this term in our hands before, in nextTE...
- Term t = new Term(Consts.FULL, first);
- for (int i=0; i<taxonomies.length; i++) {
- if (first.equals(currentOthers[i])) {
- // remember the remapping of this ordinal. Note how
- // this requires reading a posting list from the index -
- // but since we do this in lexical order of terms, just
- // like Lucene's merge works, we hope there are few seeks.
- // TODO (Facet): is there a quicker way? E.g., not specifying the
- // next term by name every time?
- TermDocs td = otherreaders[i].termDocs(t);
- td.next(); // TODO (Facet): check?
- int origordinal = td.doc();
- ordinalMaps[i].addMapping(origordinal, newordinal);
- // and move to the next category in the i'th taxonomy
- currentOthers[i] = nextTE(othertes[i]);
- if (currentOthers[i]==null) {
- otherTaxonomiesLeft--;
- }
- }
+ cp.add(term.text(), Consts.DEFAULT_DELIMITER);
+ int ordinal = findCategory(cp);
+ if (ordinal < 0) {
+ // NOTE: call addCategory so that it works well in a multi-threaded
+ // environment, in case e.g. a thread just added the category, after
+ // the findCategory() call above failed to find it.
+ ordinal = addCategory(cp);
}
- } else if (comp==0) {
- // 'first' and currentMain are the same, so both the main and some
- // other taxonomies need to be moved, but a category doesn't need
- // to be added because it already existed in the main taxonomy.
-
- // TODO (Facet): Again, is there a quicker way?
- Term t = new Term(Consts.FULL, first);
- TermDocs td = mainreader.termDocs(t);
- td.next(); // TODO (Facet): check?
- int newordinal = td.doc();
-
- currentMain = nextTE(mainte);
- for (int i=0; i<taxonomies.length; i++) {
- if (first.equals(currentOthers[i])) {
- // TODO (Facet): again, is there a quicker way?
- td = otherreaders[i].termDocs(t);
- td.next(); // TODO (Facet): check?
- int origordinal = td.doc();
- ordinalMaps[i].addMapping(origordinal, newordinal);
-
- // and move to the next category
- currentOthers[i] = nextTE(othertes[i]);
- if (currentOthers[i]==null) {
- otherTaxonomiesLeft--;
- }
- }
- }
- } else /* comp > 0 */ {
- // The currentMain doesn't appear in any of the other taxonomies -
- // we don't need to do anything, just continue to the next one
- currentMain = nextTE(mainte);
- }
- }
-
- // Close all the readers we've opened, and also tell the ordinal maps
- // we're done adding to them
- mainreader.close();
- for (int i=0; i<taxonomies.length; i++) {
- otherreaders[i].close();
- // We never actually added a mapping for the root ordinal - let's do
- // it now, just so that the map is complete (every ordinal between 0
- // and size-1 is remapped)
- ordinalMaps[i].addMapping(0, 0);
- ordinalMaps[i].addDone();
+ docs.seek(term);
+ docs.next();
+ ordinalMap.addMapping(docs.doc(), ordinal);
+ }
+ // we must add the root ordinal map, so that the map will be complete
+ // (otherwise e.g. DiskOrdinalMap may fail because it expects more
+ // categories to exist in the file).
+ ordinalMap.addMapping(0, 0);
+ ordinalMap.addDone();
+ } finally {
+ r.close();
}
}
-
+
/**
* Mapping from old ordinal to new ordinals, used when merging indexes
* wit separate taxonomies.
@@ -1071,20 +966,6 @@ public class DirectoryTaxonomyWriter imp
}
}
- private static final String nextTE(TermEnum te) throws IOException {
- if (te.next()) {
- Term t = te.term();
- // If our enumeration reached a different field, we're done. Note
- // how we're allowed compare string references, rather than the
- // actual string's contents.
- if (t.field()==Consts.FULL) {
- return t.text();
- }
- return null;
- }
- return null;
- }
-
/**
* Rollback changes to the taxonomy writer and closes the instance. Following
* this method the instance becomes unusable (calling any of its API methods
Copied: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java (from r1339027, lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomies.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java?p2=lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java&p1=lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomies.java&r1=1339027&r2=1339032&rev=1339032&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomies.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java Wed May 16 07:32:54 2012
@@ -1,20 +1,16 @@
package org.apache.lucene.facet.taxonomy.directory;
-import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
-import org.apache.lucene.store.Directory;
-import org.junit.Test;
-
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.DiskOrdinalMap;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.OrdinalMap;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -33,222 +29,197 @@ import org.apache.lucene.facet.taxonomy.
* limitations under the License.
*/
-public class TestAddTaxonomies extends LuceneTestCase {
-
- @Test
- public void test1() throws Exception {
- Directory dir1 = newDirectory();
- DirectoryTaxonomyWriter tw1 = new DirectoryTaxonomyWriter(dir1);
- tw1.addCategory(new CategoryPath("Author", "Mark Twain"));
- tw1.addCategory(new CategoryPath("Animals", "Dog"));
- Directory dir2 = newDirectory();
- DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(dir2);
- tw2.addCategory(new CategoryPath("Author", "Rob Pike"));
- tw2.addCategory(new CategoryPath("Aardvarks", "Bob"));
- tw2.close();
- Directory dir3 = newDirectory();
- DirectoryTaxonomyWriter tw3 = new DirectoryTaxonomyWriter(dir3);
- tw3.addCategory(new CategoryPath("Author", "Zebra Smith"));
- tw3.addCategory(new CategoryPath("Aardvarks", "Bob"));
- tw3.addCategory(new CategoryPath("Aardvarks", "Aaron"));
- tw3.close();
-
- MemoryOrdinalMap[] maps = new MemoryOrdinalMap[2];
- maps[0] = new MemoryOrdinalMap();
- maps[1] = new MemoryOrdinalMap();
-
- tw1.addTaxonomies(new Directory[] { dir2, dir3 }, maps);
- tw1.close();
-
- TaxonomyReader tr = new DirectoryTaxonomyReader(dir1);
-
- // Test that the merged taxonomy now contains what we expect:
- // First all the categories of the original taxonomy, in their original order:
- assertEquals(tr.getPath(0).toString(), "");
- assertEquals(tr.getPath(1).toString(), "Author");
- assertEquals(tr.getPath(2).toString(), "Author/Mark Twain");
- assertEquals(tr.getPath(3).toString(), "Animals");
- assertEquals(tr.getPath(4).toString(), "Animals/Dog");
- // Then the categories new in the new taxonomy, in alphabetical order:
- assertEquals(tr.getPath(5).toString(), "Aardvarks");
- assertEquals(tr.getPath(6).toString(), "Aardvarks/Aaron");
- assertEquals(tr.getPath(7).toString(), "Aardvarks/Bob");
- assertEquals(tr.getPath(8).toString(), "Author/Rob Pike");
- assertEquals(tr.getPath(9).toString(), "Author/Zebra Smith");
- assertEquals(tr.getSize(), 10);
-
- // Test that the maps contain what we expect
- int[] map0 = maps[0].getMap();
- assertEquals(5, map0.length);
- assertEquals(0, map0[0]);
- assertEquals(1, map0[1]);
- assertEquals(8, map0[2]);
- assertEquals(5, map0[3]);
- assertEquals(7, map0[4]);
-
- int[] map1 = maps[1].getMap();
- assertEquals(6, map1.length);
- assertEquals(0, map1[0]);
- assertEquals(1, map1[1]);
- assertEquals(9, map1[2]);
- assertEquals(5, map1[3]);
- assertEquals(7, map1[4]);
- assertEquals(6, map1[5]);
-
- tr.close();
- dir1.close();
- dir2.close();
- dir3.close();
- }
-
- // a reasonable random test
- public void testmedium() throws Exception {
- int numTests = atLeast(3);
- for (int i = 0; i < numTests; i++) {
- dotest(_TestUtil.nextInt(random, 1, 10),
- _TestUtil.nextInt(random, 1, 100),
- _TestUtil.nextInt(random, 100, 1000),
- random.nextBoolean());
- }
- }
-
- // A more comprehensive and big random test.
- @Test @Nightly
- public void testbig() throws Exception {
- dotest(2, 1000, 5000, false);
- dotest(10, 10000, 100, false);
- dotest(50, 20, 100, false);
- dotest(10, 1000, 10000, false);
- dotest(50, 20, 10000, false);
- dotest(1, 20, 10000, false);
- dotest(10, 1, 10000, false);
- dotest(10, 1000, 20000, true);
- }
-
- private void dotest(int ntaxonomies, int ncats, int range, boolean disk) throws Exception {
- Directory dirs[] = new Directory[ntaxonomies];
- Directory copydirs[] = new Directory[ntaxonomies];
+public class TestAddTaxonomy extends LuceneTestCase {
- for (int i=0; i<ntaxonomies; i++) {
+ private void dotest(int ncats, int range) throws Exception {
+ Directory dirs[] = new Directory[2];
+ for (int i = 0; i < dirs.length; i++) {
dirs[i] = newDirectory();
- copydirs[i] = newDirectory();
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[i]);
- DirectoryTaxonomyWriter copytw = new DirectoryTaxonomyWriter(copydirs[i]);
- for (int j=0; j<ncats; j++) {
+ for (int j = 0; j < ncats; j++) {
String cat = Integer.toString(random.nextInt(range));
- tw.addCategory(new CategoryPath("a",cat));
- copytw.addCategory(new CategoryPath("a",cat));
+ tw.addCategory(new CategoryPath("a", cat));
}
- // System.err.println("Taxonomy "+i+": "+tw.getSize());
tw.close();
- copytw.close();
}
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0]);
- Directory otherdirs[] = new Directory[ntaxonomies-1];
- System.arraycopy(dirs, 1, otherdirs, 0, ntaxonomies-1);
+ OrdinalMap map = randomOrdinalMap();
+ tw.addTaxonomy(dirs[1], map);
+ tw.close();
+
+ validate(dirs[0], dirs[1], map);
+
+ IOUtils.close(dirs);
+ }
+
+ private OrdinalMap randomOrdinalMap() throws IOException {
+ if (random.nextBoolean()) {
+ return new DiskOrdinalMap(_TestUtil.createTempFile("taxoMap", "", TEMP_DIR));
+ } else {
+ return new MemoryOrdinalMap();
+ }
+ }
- OrdinalMap[] maps = new OrdinalMap[ntaxonomies-1];
- if (ntaxonomies>1) {
- for (int i=0; i<ntaxonomies-1; i++) {
- if (disk) {
- // TODO: use a LTC tempfile
- maps[i] = new DiskOrdinalMap(new File(System.getProperty("java.io.tmpdir"),
- "tmpmap"+i));
- } else {
- maps[i] = new MemoryOrdinalMap();
+ private void validate(Directory dest, Directory src, OrdinalMap ordMap) throws Exception {
+ CategoryPath cp = new CategoryPath();
+ DirectoryTaxonomyReader destTR = new DirectoryTaxonomyReader(dest);
+ try {
+ final int destSize = destTR.getSize();
+ DirectoryTaxonomyReader srcTR = new DirectoryTaxonomyReader(src);
+ try {
+ int[] map = ordMap.getMap();
+
+ // validate taxo sizes
+ int srcSize = srcTR.getSize();
+ assertTrue("destination taxonomy expected to be larger than source; dest="
+ + destSize + " src=" + srcSize,
+ destSize >= srcSize);
+
+ // validate that all source categories exist in destination, and their
+ // ordinals are as expected.
+ for (int j = 1; j < srcSize; j++) {
+ srcTR.getPath(j, cp);
+ int destOrdinal = destTR.getOrdinal(cp);
+ assertTrue(cp + " not found in destination", destOrdinal > 0);
+ assertEquals(destOrdinal, map[j]);
}
+ } finally {
+ srcTR.close();
}
+ } finally {
+ destTR.close();
}
+ }
- tw.addTaxonomies(otherdirs, maps);
- // System.err.println("Merged axonomy: "+tw.getSize());
- tw.close();
+ public void testAddEmpty() throws Exception {
+ Directory dest = newDirectory();
+ DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest);
+ destTW.addCategory(new CategoryPath("Author", "Rob Pike"));
+ destTW.addCategory(new CategoryPath("Aardvarks", "Bob"));
+ destTW.commit();
+
+ Directory src = newDirectory();
+ new DirectoryTaxonomyWriter(src).close(); // create an empty taxonomy
+
+ OrdinalMap map = randomOrdinalMap();
+ destTW.addTaxonomy(src, map);
+ destTW.close();
+
+ validate(dest, src, map);
+
+ IOUtils.close(dest, src);
+ }
+
+ public void testAddToEmpty() throws Exception {
+ Directory dest = newDirectory();
+
+ Directory src = newDirectory();
+ DirectoryTaxonomyWriter srcTW = new DirectoryTaxonomyWriter(src);
+ srcTW.addCategory(new CategoryPath("Author", "Rob Pike"));
+ srcTW.addCategory(new CategoryPath("Aardvarks", "Bob"));
+ srcTW.close();
+
+ DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest);
+ OrdinalMap map = randomOrdinalMap();
+ destTW.addTaxonomy(src, map);
+ destTW.close();
+
+ validate(dest, src, map);
+
+ IOUtils.close(dest, src);
+ }
+
+ // A more comprehensive and big random test.
+ @Nightly
+ public void testBig() throws Exception {
+ dotest(200, 10000);
+ dotest(1000, 20000);
+ // really big
+ dotest(400000, 1000000);
+ }
- // Check that all original categories in the main taxonomy remain in
- // unchanged, and the rest of the taxonomies are completely unchanged.
- for (int i=0; i<ntaxonomies; i++) {
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[i]);
- TaxonomyReader copytr = new DirectoryTaxonomyReader(copydirs[i]);
- if (i==0) {
- assertTrue(tr.getSize() >= copytr.getSize());
- } else {
- assertEquals(copytr.getSize(), tr.getSize());
- }
- for (int j=0; j<copytr.getSize(); j++) {
- String expected = copytr.getPath(j).toString();
- String got = tr.getPath(j).toString();
- assertTrue("Comparing category "+j+" of taxonomy "+i+": expected "+expected+", got "+got,
- expected.equals(got));
- }
- tr.close();
- copytr.close();
+ // a reasonable random test
+ public void testMedium() throws Exception {
+ int numTests = atLeast(3);
+ for (int i = 0; i < numTests; i++) {
+ dotest(_TestUtil.nextInt(random, 2, 100),
+ _TestUtil.nextInt(random, 100, 1000));
}
+ }
+
+ public void testSimple() throws Exception {
+ Directory dest = newDirectory();
+ DirectoryTaxonomyWriter tw1 = new DirectoryTaxonomyWriter(dest);
+ tw1.addCategory(new CategoryPath("Author", "Mark Twain"));
+ tw1.addCategory(new CategoryPath("Animals", "Dog"));
+ tw1.addCategory(new CategoryPath("Author", "Rob Pike"));
+
+ Directory src = newDirectory();
+ DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(src);
+ tw2.addCategory(new CategoryPath("Author", "Rob Pike"));
+ tw2.addCategory(new CategoryPath("Aardvarks", "Bob"));
+ tw2.close();
- // Check that all the new categories in the main taxonomy are in
- // lexicographic order. This isn't a requirement of our API, but happens
- // this way in our current implementation.
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0]);
- TaxonomyReader copytr = new DirectoryTaxonomyReader(copydirs[0]);
- if (tr.getSize() > copytr.getSize()) {
- String prev = tr.getPath(copytr.getSize()).toString();
- for (int j=copytr.getSize()+1; j<tr.getSize(); j++) {
- String n = tr.getPath(j).toString();
- assertTrue(prev.compareTo(n)<0);
- prev=n;
- }
- }
- int oldsize = copytr.getSize(); // remember for later
- tr.close();
- copytr.close();
-
- // Check that all the categories from other taxonomies exist in the new
- // taxonomy.
- TaxonomyReader main = new DirectoryTaxonomyReader(dirs[0]);
- for (int i=1; i<ntaxonomies; i++) {
- TaxonomyReader other = new DirectoryTaxonomyReader(dirs[i]);
- for (int j=0; j<other.getSize(); j++) {
- int otherord = main.getOrdinal(other.getPath(j));
- assertTrue(otherord != TaxonomyReader.INVALID_ORDINAL);
- }
- other.close();
- }
+ OrdinalMap map = randomOrdinalMap();
- // Check that all the new categories in the merged taxonomy exist in
- // one of the added taxonomies.
- TaxonomyReader[] others = new TaxonomyReader[ntaxonomies-1];
- for (int i=1; i<ntaxonomies; i++) {
- others[i-1] = new DirectoryTaxonomyReader(dirs[i]);
- }
- for (int j=oldsize; j<main.getSize(); j++) {
- boolean found=false;
- CategoryPath path = main.getPath(j);
- for (int i=1; i<ntaxonomies; i++) {
- if (others[i-1].getOrdinal(path) != TaxonomyReader.INVALID_ORDINAL) {
- found=true;
- break;
- }
- }
- if (!found) {
- fail("Found category "+j+" ("+path+") in merged taxonomy not in any of the separate ones");
- }
- }
+ tw1.addTaxonomy(src, map);
+ tw1.close();
- // Check that all the maps are correct
- for (int i=0; i<ntaxonomies-1; i++) {
- int[] map = maps[i].getMap();
- for (int j=0; j<map.length; j++) {
- assertEquals(map[j], main.getOrdinal(others[i].getPath(j)));
- }
- }
+ validate(dest, src, map);
+
+ IOUtils.close(dest, src);
+ }
- for (int i=1; i<ntaxonomies; i++) {
- others[i-1].close();
+ public void testConcurrency() throws Exception {
+ // tests that addTaxonomy and addCategory work in parallel
+ final int numCategories = atLeast(5000);
+
+ // build an input taxonomy index
+ Directory src = newDirectory();
+ DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(src);
+ for (int i = 0; i < numCategories; i++) {
+ tw.addCategory(new CategoryPath("a", Integer.toString(i)));
}
-
- main.close();
- IOUtils.close(dirs);
- IOUtils.close(copydirs);
+ tw.close();
+
+ // now add the taxonomy to an empty taxonomy, while adding the categories
+ // again, in parallel -- in the end, no duplicate categories should exist.
+ Directory dest = newDirectory();
+ final DirectoryTaxonomyWriter destTW = new DirectoryTaxonomyWriter(dest);
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ for (int i = 0; i < numCategories; i++) {
+ try {
+ destTW.addCategory(new CategoryPath("a", Integer.toString(i)));
+ } catch (IOException e) {
+ // shouldn't happen - if it does, let the test fail on uncaught exception.
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+ t.start();
+
+ OrdinalMap map = new MemoryOrdinalMap();
+ destTW.addTaxonomy(src, map);
+ t.join();
+ destTW.close();
+
+ // now validate
+
+ DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dest);
+ // +2 to account for the root category + "a"
+ assertEquals(numCategories + 2, dtr.getSize());
+ HashSet<CategoryPath> categories = new HashSet<CategoryPath>();
+ for (int i = 1; i < dtr.getSize(); i++) {
+ CategoryPath cat = dtr.getPath(i);
+ assertTrue("category " + cat + " already existed", categories.add(cat));
+ }
+ dtr.close();
+
+ IOUtils.close(src, dest);
}
-
+
}