You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2012/02/16 13:54:58 UTC
svn commit: r1244964 - in /lucene/dev/trunk: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/
lucene/core/src/java/org/apache/lucene/codecs/lucene3x/
modules/analysis/kur...
Author: shaie
Date: Thu Feb 16 12:54:56 2012
New Revision: 1244964
URL: http://svn.apache.org/viewvc?rev=1244964&view=rev
Log:
LUCENE-3794: DirectoryTaxonomyWriter can lose the INDEX_CREATE_TIME property, causing DirTaxoReader.refresh() to falsely succeed (or fail)
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/trunk/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (props changed)
lucene/dev/trunk/modules/analysis/kuromoji/ (props changed)
lucene/dev/trunk/modules/benchmark/ (props changed)
lucene/dev/trunk/modules/facet/ (props changed)
lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/trunk/solr/ (props changed)
lucene/dev/trunk/solr/CHANGES.txt (props changed)
lucene/dev/trunk/solr/LICENSE.txt (props changed)
lucene/dev/trunk/solr/NOTICE.txt (props changed)
lucene/dev/trunk/solr/README.txt (props changed)
lucene/dev/trunk/solr/build.xml (props changed)
lucene/dev/trunk/solr/client/ (props changed)
lucene/dev/trunk/solr/common-build.xml (props changed)
lucene/dev/trunk/solr/contrib/ (props changed)
lucene/dev/trunk/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/trunk/solr/contrib/uima/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/trunk/solr/core/ (props changed)
lucene/dev/trunk/solr/core/src/java/ (props changed)
lucene/dev/trunk/solr/core/src/test/ (props changed)
lucene/dev/trunk/solr/dev-tools/ (props changed)
lucene/dev/trunk/solr/example/ (props changed)
lucene/dev/trunk/solr/lib/ (props changed)
lucene/dev/trunk/solr/scripts/ (props changed)
lucene/dev/trunk/solr/site/ (props changed)
lucene/dev/trunk/solr/site-src/ (props changed)
lucene/dev/trunk/solr/solrj/ (props changed)
lucene/dev/trunk/solr/solrj/src/java/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/trunk/solr/test-framework/ (props changed)
lucene/dev/trunk/solr/testlogging.properties (props changed)
lucene/dev/trunk/solr/webapp/ (props changed)
Modified: lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1244964&r1=1244963&r2=1244964&view=diff
==============================================================================
--- lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Thu Feb 16 12:54:56 2012
@@ -32,6 +32,7 @@ import org.apache.lucene.index.LogByteSi
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.AlreadyClosedException;
@@ -87,13 +88,14 @@ import org.apache.lucene.facet.taxonomy.
public class DirectoryTaxonomyWriter implements TaxonomyWriter {
/**
- * Property name of user commit data that contains the creation time of a taxonomy index.
+ * Property name of user commit data that contains the creation time of a
+ * taxonomy index.
* <p>
- * Applications making use of {@link TaxonomyWriter#commit(Map)} should not use this
- * particular property name.
+ * Applications should not use this property in their commit data because it
+ * will be overridden by this taxonomy writer.
*/
public static final String INDEX_CREATE_TIME = "index.create.time";
-
+
private IndexWriter indexWriter;
private int nextID;
private char delimiter = Consts.DEFAULT_DELIMITER;
@@ -116,11 +118,15 @@ public class DirectoryTaxonomyWriter imp
private DirectoryReader reader;
private int cacheMisses;
- /**
- * When a taxonomy is created, we mark that its create time should be committed in the
- * next commit.
- */
- private String taxoIndexCreateTime = null;
+ /** Records the taxonomy index creation time. */
+ private final String createTime;
+
+ /** Reads the commit data from a Directory. */
+ private static Map<String, String> readCommitData(Directory dir) throws IOException {
+ SegmentInfos infos = new SegmentInfos();
+ infos.read(dir);
+ return infos.getUserData();
+ }
/**
* setDelimiter changes the character that the taxonomy uses in its internal
@@ -185,12 +191,20 @@ public class DirectoryTaxonomyWriter imp
* if another error occurred.
*/
public DirectoryTaxonomyWriter(Directory directory, OpenMode openMode,
- TaxonomyWriterCache cache)
- throws CorruptIndexException, LockObtainFailedException,
- IOException {
+ TaxonomyWriterCache cache) throws IOException {
if (!DirectoryReader.indexExists(directory) || openMode==OpenMode.CREATE) {
- taxoIndexCreateTime = Long.toString(System.nanoTime());
+ createTime = Long.toString(System.nanoTime());
+ } else {
+ Map<String, String> commitData = readCommitData(directory);
+ if (commitData != null) {
+ // It is ok if an existing index doesn't have commitData, or the
+ // INDEX_CREATE_TIME property. If ever it will be recreated, we'll set
+ // createTime accordingly in the above 'if'.
+ createTime = commitData.get(INDEX_CREATE_TIME);
+ } else {
+ createTime = null;
+ }
}
IndexWriterConfig config = createIndexWriterConfig(openMode);
@@ -209,7 +223,7 @@ public class DirectoryTaxonomyWriter imp
this.nextID = indexWriter.maxDoc();
- if (cache==null) {
+ if (cache == null) {
cache = defaultTaxonomyWriterCache();
}
this.cache = cache;
@@ -323,10 +337,7 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void close() throws CorruptIndexException, IOException {
if (indexWriter != null) {
- if (taxoIndexCreateTime != null) {
- indexWriter.commit(combinedCommitData(null));
- taxoIndexCreateTime = null;
- }
+ indexWriter.commit(combinedCommitData(null));
doClose();
}
}
@@ -636,12 +647,7 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void commit() throws CorruptIndexException, IOException {
ensureOpen();
- if (taxoIndexCreateTime != null) {
- indexWriter.commit(combinedCommitData(null));
- taxoIndexCreateTime = null;
- } else {
- indexWriter.commit();
- }
+ indexWriter.commit(combinedCommitData(null));
refreshReader();
}
@@ -653,7 +659,9 @@ public class DirectoryTaxonomyWriter imp
if (userData != null) {
m.putAll(userData);
}
- m.put(INDEX_CREATE_TIME, taxoIndexCreateTime);
+ if (createTime != null) {
+ m.put(INDEX_CREATE_TIME, createTime);
+ }
return m;
}
@@ -665,12 +673,7 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void commit(Map<String,String> commitUserData) throws CorruptIndexException, IOException {
ensureOpen();
- if (taxoIndexCreateTime != null) {
- indexWriter.commit(combinedCommitData(commitUserData));
- taxoIndexCreateTime = null;
- } else {
- indexWriter.commit(commitUserData);
- }
+ indexWriter.commit(combinedCommitData(commitUserData));
refreshReader();
}
@@ -681,12 +684,7 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void prepareCommit() throws CorruptIndexException, IOException {
ensureOpen();
- if (taxoIndexCreateTime != null) {
- indexWriter.prepareCommit(combinedCommitData(null));
- taxoIndexCreateTime = null;
- } else {
- indexWriter.prepareCommit();
- }
+ indexWriter.prepareCommit(combinedCommitData(null));
}
/**
@@ -696,12 +694,7 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void prepareCommit(Map<String,String> commitUserData) throws CorruptIndexException, IOException {
ensureOpen();
- if (taxoIndexCreateTime != null) {
- indexWriter.prepareCommit(combinedCommitData(commitUserData));
- taxoIndexCreateTime = null;
- } else {
- indexWriter.prepareCommit(commitUserData);
- }
+ indexWriter.prepareCommit(combinedCommitData(commitUserData));
}
/**
@@ -864,7 +857,7 @@ public class DirectoryTaxonomyWriter imp
TermsEnum[] othertes = new TermsEnum[taxonomies.length];
DocsEnum[] otherdocsEnum = new DocsEnum[taxonomies.length]; // just for reuse
for (int i=0; i<taxonomies.length; i++) {
- otherreaders[i] = IndexReader.open(taxonomies[i]);
+ otherreaders[i] = DirectoryReader.open(taxonomies[i]);
terms = MultiFields.getTerms(otherreaders[i], Consts.FULL);
assert terms != null; // TODO (Facet): explicit check / throw exception?
othertes[i] = terms.iterator(null);
Modified: lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java?rev=1244964&r1=1244963&r2=1244964&view=diff
==============================================================================
--- lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java (original)
+++ lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java Thu Feb 16 12:54:56 2012
@@ -151,7 +151,7 @@ public class TestDirectoryTaxonomyReader
tw.close();
tr = new DirectoryTaxonomyReader(dir);
- int baseNumcategories = tr.getSize();
+ int baseNumCategories = tr.getSize();
for (int i=0; i<n; i++) {
int k = random.nextInt(n);
@@ -172,7 +172,7 @@ public class TestDirectoryTaxonomyReader
tr = new DirectoryTaxonomyReader(dir);
}
}
- assertEquals("Wrong #categories in taxonomy (i="+i+", k="+k+")", baseNumcategories + 1 + k, tr.getSize());
+ assertEquals("Wrong #categories in taxonomy (i="+i+", k="+k+")", baseNumCategories + 1 + k, tr.getSize());
}
} finally {
IOUtils.close(tr, tw, dir);
Modified: lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java?rev=1244964&r1=1244963&r2=1244964&view=diff
==============================================================================
--- lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java (original)
+++ lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java Thu Feb 16 12:54:56 2012
@@ -1,19 +1,21 @@
package org.apache.lucene.facet.taxonomy.directory;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.InconsistentTaxonomyException;
+import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
-import org.junit.Test;
-
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
+import org.junit.Test;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -59,7 +61,7 @@ public class TestDirectoryTaxonomyWriter
ltw.commit(); // first commit, so that an index will be created
ltw.addCategory(new CategoryPath("a"));
- IndexReader r = IndexReader.open(dir);
+ IndexReader r = DirectoryReader.open(dir);
assertEquals("No categories should have been committed to the underlying directory", 1, r.numDocs());
r.close();
ltw.close();
@@ -68,23 +70,38 @@ public class TestDirectoryTaxonomyWriter
@Test
public void testCommitUserData() throws Exception {
- // Verifies that committed data is retrievable
+ // Verifies taxonomy commit data
Directory dir = newDirectory();
- DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
- assertFalse(DirectoryReader.indexExists(dir));
- ltw.commit(); // first commit, so that an index will be created
- ltw.addCategory(new CategoryPath("a"));
- ltw.addCategory(new CategoryPath("b"));
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
+ taxoWriter.addCategory(new CategoryPath("a"));
+ taxoWriter.addCategory(new CategoryPath("b"));
Map <String, String> userCommitData = new HashMap<String, String>();
userCommitData.put("testing", "1 2 3");
- ltw.commit(userCommitData);
- ltw.close();
- DirectoryReader r = IndexReader.open(dir);
+ taxoWriter.commit(userCommitData);
+ taxoWriter.close();
+ DirectoryReader r = DirectoryReader.open(dir);
assertEquals("2 categories plus root should have been committed to the underlying directory", 3, r.numDocs());
Map <String, String> readUserCommitData = r.getIndexCommit().getUserData();
assertTrue("wrong value extracted from commit data",
"1 2 3".equals(readUserCommitData.get("testing")));
+ assertNotNull("index.create.time not found in commitData", readUserCommitData.get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME));
r.close();
+
+ // open DirTaxoWriter again and commit, INDEX_CREATE_TIME should still exist
+ // in the commit data, otherwise DirTaxoReader.refresh() might not detect
+ // that the taxonomy index has been recreated.
+ taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
+ taxoWriter.addCategory(new CategoryPath("c")); // add a category so that commit will happen
+ taxoWriter.commit(new HashMap<String, String>(){{
+ put("just", "data");
+ }});
+ taxoWriter.close();
+
+ r = DirectoryReader.open(dir);
+ readUserCommitData = r.getIndexCommit().getUserData();
+ assertNotNull("index.create.time not found in commitData", readUserCommitData.get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME));
+ r.close();
+
dir.close();
}
@@ -118,5 +135,71 @@ public class TestDirectoryTaxonomyWriter
}
dir.close();
}
+
+ private void touchTaxo(DirectoryTaxonomyWriter taxoWriter, CategoryPath cp) throws IOException {
+ taxoWriter.addCategory(cp);
+ taxoWriter.commit(new HashMap<String, String>(){{
+ put("just", "data");
+ }});
+ }
+
+ @Test
+ public void testRecreateAndRefresh() throws Exception {
+ // DirTaxoWriter lost the INDEX_CREATE_TIME property if it was opened in
+ // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
+ // DirTaxoReader succeeding to refresh().
+ Directory dir = newDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
+ touchTaxo(taxoWriter, new CategoryPath("a"));
+
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
+
+ touchTaxo(taxoWriter, new CategoryPath("b"));
+
+ // this should not fail
+ taxoReader.refresh();
+
+ // now recreate the taxonomy, and check that the timestamp is preserved after opening DirTW again.
+ taxoWriter.close();
+ taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, new NoOpCache());
+ touchTaxo(taxoWriter, new CategoryPath("c"));
+ taxoWriter.close();
+
+ taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
+ touchTaxo(taxoWriter, new CategoryPath("d"));
+ taxoWriter.close();
+
+ // this should fail
+ try {
+ taxoReader.refresh();
+ fail("IconsistentTaxonomyException should have been thrown");
+ } catch (InconsistentTaxonomyException e) {
+ // ok, expected
+ }
+
+ taxoReader.close();
+ dir.close();
+ }
+
+ @Test
+ public void testUndefinedCreateTime() throws Exception {
+ // tests that if the taxonomy index doesn't have the INDEX_CREATE_TIME
+ // property (supports pre-3.6 indexes), all still works.
+ Directory dir = newDirectory();
+
+ // create an empty index first, so that DirTaxoWriter initializes createTime to null.
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
+ // we cannot commit null keys/values, this ensures that if DirTW.createTime is null, we can still commit.
+ taxoWriter.close();
+
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
+ taxoReader.refresh();
+ taxoReader.close();
+
+ dir.close();
+ }
}