You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2012/05/16 10:01:41 UTC
svn commit: r1339047 - in /lucene/dev/trunk/lucene: ./ facet/
facet/src/examples/org/apache/lucene/facet/example/merge/
facet/src/java/org/apache/lucene/facet/taxonomy/directory/
facet/src/test/org/apache/lucene/facet/taxonomy/directory/
Author: shaie
Date: Wed May 16 08:01:40 2012
New Revision: 1339047
URL: http://svn.apache.org/viewvc?rev=1339047&view=rev
Log:
LUCENE-4060: port to trunk
Added:
lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java
- copied, changed from r1339032, lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java
Removed:
lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomies.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/facet/ (props changed)
lucene/dev/trunk/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java
lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1339047&r1=1339046&r2=1339047&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Wed May 16 08:01:40 2012
@@ -953,6 +953,11 @@ Bug fixes
offset calculation in PathHierarchyTokenizer.
(Mike McCandless, Uwe Schindler, Robert Muir)
+* LUCENE-4060: Fix a synchronization bug in
+ DirectoryTaxonomyWriter.addTaxonomies(). Also, the method has been renamed to
+ addTaxonomy and now takes only one Directory and one OrdinalMap.
+ (Shai Erera, Gilad Barkai)
+
Documentation
* LUCENE-3958: Javadocs corrections for IndexWriter.
Modified: lucene/dev/trunk/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java?rev=1339047&r1=1339046&r2=1339047&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java (original)
+++ lucene/dev/trunk/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java Wed May 16 08:01:40 2012
@@ -81,7 +81,7 @@ public class TaxonomyMergeUtils {
OrdinalMap map, IndexWriter destIndexWriter,
DirectoryTaxonomyWriter destTaxWriter) throws IOException {
// merge the taxonomies
- destTaxWriter.addTaxonomies(new Directory[] { srcTaxDir }, new OrdinalMap[] { map });
+ destTaxWriter.addTaxonomy(srcTaxDir, map);
PayloadProcessorProvider payloadProcessor = new FacetsPayloadProcessorProvider(
srcIndexDir, map.getMap(), new DefaultFacetIndexingParams());
Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java?rev=1339047&r1=1339046&r2=1339047&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java (original)
+++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/Consts.java Wed May 16 08:01:40 2012
@@ -4,8 +4,6 @@ import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.index.StoredFieldVisitor.Status;
-import org.apache.lucene.store.IndexInput;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -42,6 +40,7 @@ abstract class Consts {
public static final class LoadFullPathOnly extends StoredFieldVisitor {
private String fullPath;
+ @Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
fullPath = value;
}
Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1339047&r1=1339046&r2=1339047&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Wed May 16 08:01:40 2012
@@ -12,15 +12,21 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.cl2o.Cl2oTaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.lru.LruTaxonomyWriterCache;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
@@ -30,9 +36,9 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.LogByteSizeMergePolicy;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.AlreadyClosedException;
@@ -44,13 +50,6 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
-import org.apache.lucene.facet.taxonomy.writercache.cl2o.Cl2oTaxonomyWriterCache;
-import org.apache.lucene.facet.taxonomy.writercache.lru.LruTaxonomyWriterCache;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -812,6 +811,7 @@ public class DirectoryTaxonomyWriter imp
}
return parentArray;
}
+
@Override
public int getParent(int ordinal) throws IOException {
ensureOpen();
@@ -823,158 +823,47 @@ public class DirectoryTaxonomyWriter imp
}
return getParentArray().getArray()[ordinal];
}
-
+
/**
- * Take all the categories of one or more given taxonomies, and add them to
- * the main taxonomy (this), if they are not already there.
- * <P>
- * Additionally, fill a <I>mapping</I> for each of the added taxonomies,
- * mapping its ordinals to the ordinals in the enlarged main taxonomy.
- * These mapping are saved into an array of OrdinalMap objects given by the
- * user, one for each of the given taxonomies (not including "this", the main
- * taxonomy). Often the first of these will be a MemoryOrdinalMap and the
- * others will be a DiskOrdinalMap - see discussion in {OrdinalMap}.
- * <P>
- * Note that the taxonomies to be added are given as Directory objects,
- * not opened TaxonomyReader/TaxonomyWriter objects, so if any of them are
- * currently managed by an open TaxonomyWriter, make sure to commit() (or
- * close()) it first. The main taxonomy (this) is an open TaxonomyWriter,
- * and does not need to be commit()ed before this call.
+ * Takes the categories from the given taxonomy directory, and adds the
+ * missing ones to this taxonomy. Additionally, it fills the given
+ * {@link OrdinalMap} with a mapping from the original ordinal to the new
+ * ordinal.
*/
- public void addTaxonomies(Directory[] taxonomies, OrdinalMap[] ordinalMaps) throws IOException {
+ public void addTaxonomy(Directory taxoDir, OrdinalMap map) throws IOException {
ensureOpen();
- // To prevent us stepping on the rest of this class's decisions on when
- // to open a reader, and when not, we'll be opening a new reader instead
- // of using the existing "reader" object:
- IndexReader mainreader = openReader();
- // TODO (Facet): can this then go segment-by-segment and avoid MultiDocsEnum etc?
- Terms terms = MultiFields.getTerms(mainreader, Consts.FULL);
- assert terms != null; // TODO (Facet): explicit check / throw exception?
- TermsEnum mainte = terms.iterator(null);
- DocsEnum mainde = null;
-
- IndexReader[] otherreaders = new IndexReader[taxonomies.length];
- TermsEnum[] othertes = new TermsEnum[taxonomies.length];
- DocsEnum[] otherdocsEnum = new DocsEnum[taxonomies.length]; // just for reuse
- for (int i=0; i<taxonomies.length; i++) {
- otherreaders[i] = DirectoryReader.open(taxonomies[i]);
- terms = MultiFields.getTerms(otherreaders[i], Consts.FULL);
- assert terms != null; // TODO (Facet): explicit check / throw exception?
- othertes[i] = terms.iterator(null);
- // Also tell the ordinal maps their expected sizes:
- ordinalMaps[i].setSize(otherreaders[i].numDocs());
- }
-
- CategoryPath cp = new CategoryPath();
-
- // We keep a "current" cursor over the alphabetically-ordered list of
- // categories in each taxonomy. We start the cursor on the first
- // (alphabetically) category of each taxonomy:
-
- String currentMain;
- String[] currentOthers = new String[taxonomies.length];
- currentMain = nextTE(mainte);
- int otherTaxonomiesLeft = 0;
- for (int i=0; i<taxonomies.length; i++) {
- currentOthers[i] = nextTE(othertes[i]);
- if (currentOthers[i]!=null) {
- otherTaxonomiesLeft++;
- }
- }
-
- // And then, at each step look at the first (alphabetically) of the
- // current taxonomies.
- // NOTE: The most efficient way we could have done this is using a
- // PriorityQueue. But for simplicity, and assuming that usually we'll
- // have a very small number of other taxonomies (often just 1), we use
- // a more naive algorithm (o(ntaxonomies) instead of o(ln ntaxonomies)
- // per step)
-
- while (otherTaxonomiesLeft>0) {
- // TODO: use a pq here
- String first=null;
- for (int i=0; i<taxonomies.length; i++) {
- if (currentOthers[i]==null) continue;
- if (first==null || first.compareTo(currentOthers[i])>0) {
- first = currentOthers[i];
- }
- }
- int comp = 0;
- if (currentMain==null || (comp = currentMain.compareTo(first))>0) {
- // If 'first' is before currentMain, or currentMain is null,
- // then 'first' is a new category and we need to add it to the
- // main taxonomy. Then for all taxonomies with this 'first'
- // category, we need to add the new category number to their
- // map, and move to the next category in all of them.
+ IndexReader r = DirectoryReader.open(taxoDir);
+ try {
+ final int size = r.numDocs();
+ final OrdinalMap ordinalMap = map;
+ ordinalMap.setSize(size);
+ CategoryPath cp = new CategoryPath();
+ Terms terms = MultiFields.getTerms(r, Consts.FULL);
+ TermsEnum te = terms.iterator(null);
+ Bits liveDocs = MultiFields.getLiveDocs(r);
+ DocsEnum docs = null;
+ // we call next() first, to skip the root category which always exists.
+ while (te.next() != null) {
+ String value = te.term().utf8ToString();
cp.clear();
- cp.add(first, delimiter);
- // We can call internalAddCategory() instead of addCategory()
- // because we know the category hasn't been seen yet.
- int newordinal = internalAddCategory(cp, cp.length());
- // TODO (Facet): we already had this term in our hands before, in nextTE...
- // // TODO (Facet): no need to make this term?
- for (int i=0; i<taxonomies.length; i++) {
- if (first.equals(currentOthers[i])) {
- // remember the remapping of this ordinal. Note how
- // this requires reading a posting list from the index -
- // but since we do this in lexical order of terms, just
- // like Lucene's merge works, we hope there are few seeks.
- // TODO (Facet): is there a quicker way? E.g., not specifying the
- // next term by name every time?
- otherdocsEnum[i] = othertes[i].docs(MultiFields.getLiveDocs(otherreaders[i]), otherdocsEnum[i], false);
- otherdocsEnum[i].nextDoc(); // TODO (Facet): check?
- int origordinal = otherdocsEnum[i].docID();
- ordinalMaps[i].addMapping(origordinal, newordinal);
- // and move to the next category in the i'th taxonomy
- currentOthers[i] = nextTE(othertes[i]);
- if (currentOthers[i]==null) {
- otherTaxonomiesLeft--;
- }
- }
- }
- } else if (comp==0) {
- // 'first' and currentMain are the same, so both the main and some
- // other taxonomies need to be moved, but a category doesn't need
- // to be added because it already existed in the main taxonomy.
-
- // TODO (Facet): Again, is there a quicker way?
- mainde = mainte.docs(MultiFields.getLiveDocs(mainreader), mainde, false);
- mainde.nextDoc(); // TODO (Facet): check?
- int newordinal = mainde.docID();
-
- currentMain = nextTE(mainte);
- for (int i=0; i<taxonomies.length; i++) {
- if (first.equals(currentOthers[i])) {
- // TODO (Facet): again, is there a quicker way?
- otherdocsEnum[i] = othertes[i].docs(MultiFields.getLiveDocs(otherreaders[i]), otherdocsEnum[i], false);
- otherdocsEnum[i].nextDoc(); // TODO (Facet): check?
- int origordinal = otherdocsEnum[i].docID();
- ordinalMaps[i].addMapping(origordinal, newordinal);
-
- // and move to the next category
- currentOthers[i] = nextTE(othertes[i]);
- if (currentOthers[i]==null) {
- otherTaxonomiesLeft--;
- }
- }
+ cp.add(value, Consts.DEFAULT_DELIMITER);
+ int ordinal = findCategory(cp);
+ if (ordinal < 0) {
+ // NOTE: call addCategory so that it works well in a multi-threaded
+ // environment, in case e.g. a thread just added the category, after
+ // the findCategory() call above failed to find it.
+ ordinal = addCategory(cp);
}
- } else /* comp > 0 */ {
- // The currentMain doesn't appear in any of the other taxonomies -
- // we don't need to do anything, just continue to the next one
- currentMain = nextTE(mainte);
+ docs = te.docs(liveDocs, docs, false);
+ ordinalMap.addMapping(docs.nextDoc(), ordinal);
}
- }
-
- // Close all the readers we've opened, and also tell the ordinal maps
- // we're done adding to them
- mainreader.close();
- for (int i=0; i<taxonomies.length; i++) {
- otherreaders[i].close();
- // We never actually added a mapping for the root ordinal - let's do
- // it now, just so that the map is complete (every ordinal between 0
- // and size-1 is remapped)
- ordinalMaps[i].addMapping(0, 0);
- ordinalMaps[i].addDone();
+ // we must add the root ordinal map, so that the map will be complete
+ // (otherwise e.g. DiskOrdinalMap may fail because it expects more
+ // categories to exist in the file).
+ ordinalMap.addMapping(0, 0);
+ ordinalMap.addDone();
+ } finally {
+ r.close();
}
}
@@ -1113,13 +1002,6 @@ public class DirectoryTaxonomyWriter imp
}
}
- private static final String nextTE(TermsEnum te) throws IOException {
- if (te.next() != null) {
- return te.term().utf8ToString(); // TODO (Facet): avoid String creation/use Bytes?
- }
- return null;
- }
-
/**
* Rollback changes to the taxonomy writer and closes the instance. Following
* this method the instance becomes unusable (calling any of its API methods
Copied: lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java (from r1339032, lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java?p2=lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java&p1=lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java&r1=1339032&r2=1339047&rev=1339047&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java (original)
+++ lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestAddTaxonomy.java Wed May 16 08:01:40 2012
@@ -2,6 +2,7 @@ package org.apache.lucene.facet.taxonomy
import java.io.IOException;
import java.util.HashSet;
+import java.util.Random;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.DiskOrdinalMap;
@@ -33,6 +34,7 @@ public class TestAddTaxonomy extends Luc
private void dotest(int ncats, int range) throws Exception {
Directory dirs[] = new Directory[2];
+ Random random = random();
for (int i = 0; i < dirs.length; i++) {
dirs[i] = newDirectory();
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[i]);
@@ -54,7 +56,7 @@ public class TestAddTaxonomy extends Luc
}
private OrdinalMap randomOrdinalMap() throws IOException {
- if (random.nextBoolean()) {
+ if (random().nextBoolean()) {
return new DiskOrdinalMap(_TestUtil.createTempFile("taxoMap", "", TEMP_DIR));
} else {
return new MemoryOrdinalMap();
@@ -141,6 +143,7 @@ public class TestAddTaxonomy extends Luc
// a reasonable random test
public void testMedium() throws Exception {
+ Random random = random();
int numTests = atLeast(3);
for (int i = 0; i < numTests; i++) {
dotest(_TestUtil.nextInt(random, 2, 100),