You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/23 18:53:37 UTC
svn commit: r1126579 - in /lucene/dev/branches/flexscoring: ./
dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ant/
dev-tools/idea/lucene/contrib/db/ dev-tools/idea/lucene/contrib/lucli/
dev-tools/idea/lucene/contrib/swing/ dev-tools/maven/ dev-too...
Author: rmuir
Date: Mon May 23 16:53:35 2011
New Revision: 1126579
URL: http://svn.apache.org/viewvc?rev=1126579&view=rev
Log:
merge trunk (1126282:1126565)
Added:
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
- copied unchanged from r1126565, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
Removed:
lucene/dev/branches/flexscoring/dev-tools/idea/lucene/contrib/ant/
lucene/dev/branches/flexscoring/dev-tools/idea/lucene/contrib/db/
lucene/dev/branches/flexscoring/dev-tools/idea/lucene/contrib/lucli/
lucene/dev/branches/flexscoring/dev-tools/idea/lucene/contrib/swing/
lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/ant/
lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/db/
lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/lucli/
lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/swing/
Modified:
lucene/dev/branches/flexscoring/ (props changed)
lucene/dev/branches/flexscoring/dev-tools/idea/.idea/ant.xml
lucene/dev/branches/flexscoring/dev-tools/idea/.idea/modules.xml
lucene/dev/branches/flexscoring/dev-tools/idea/.idea/workspace.xml
lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/pom.xml.template
lucene/dev/branches/flexscoring/dev-tools/maven/pom.xml.template
lucene/dev/branches/flexscoring/lucene/ (props changed)
lucene/dev/branches/flexscoring/lucene/CHANGES.txt
lucene/dev/branches/flexscoring/lucene/backwards/ (props changed)
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java
lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
lucene/dev/branches/flexscoring/solr/ (props changed)
lucene/dev/branches/flexscoring/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/flexscoring/solr/LICENSE.txt (props changed)
lucene/dev/branches/flexscoring/solr/NOTICE.txt (props changed)
lucene/dev/branches/flexscoring/solr/README.txt (props changed)
lucene/dev/branches/flexscoring/solr/build.xml (props changed)
lucene/dev/branches/flexscoring/solr/client/ (props changed)
lucene/dev/branches/flexscoring/solr/common-build.xml (props changed)
lucene/dev/branches/flexscoring/solr/contrib/ (props changed)
lucene/dev/branches/flexscoring/solr/example/ (props changed)
lucene/dev/branches/flexscoring/solr/lib/ (props changed)
lucene/dev/branches/flexscoring/solr/site/ (props changed)
lucene/dev/branches/flexscoring/solr/src/ (props changed)
lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/FileFloatSource.java
lucene/dev/branches/flexscoring/solr/src/test-files/solr/conf/solrconfig-functionquery.xml
lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
lucene/dev/branches/flexscoring/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/flexscoring/dev-tools/idea/.idea/ant.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/dev-tools/idea/.idea/ant.xml?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/dev-tools/idea/.idea/ant.xml (original)
+++ lucene/dev/branches/flexscoring/dev-tools/idea/.idea/ant.xml Mon May 23 16:53:35 2011
@@ -3,21 +3,15 @@
<component name="AntConfiguration">
<buildFile url="file://$PROJECT_DIR$/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/build.xml" />
- <buildFile url="file://$PROJECT_DIR$/lucene/contrib/ant/build.xml" />
- <buildFile url="file://$PROJECT_DIR$/lucene/contrib/db/bdb/build.xml" />
- <buildFile url="file://$PROJECT_DIR$/lucene/contrib/db/bdb-je/build.xml" />
- <buildFile url="file://$PROJECT_DIR$/lucene/contrib/db/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/demo/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/highlighter/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/instantiated/build.xml" />
- <buildFile url="file://$PROJECT_DIR$/lucene/contrib/lucli/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/memory/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/misc/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/queries/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/queryparser/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/spatial/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/spellchecker/build.xml" />
- <buildFile url="file://$PROJECT_DIR$/lucene/contrib/swing/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/wordnet/build.xml" />
<buildFile url="file://$PROJECT_DIR$/lucene/contrib/xml-query-parser/build.xml" />
<buildFile url="file://$PROJECT_DIR$/modules/analysis/common/build.xml" />
Modified: lucene/dev/branches/flexscoring/dev-tools/idea/.idea/modules.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/dev-tools/idea/.idea/modules.xml?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/dev-tools/idea/.idea/modules.xml (original)
+++ lucene/dev/branches/flexscoring/dev-tools/idea/.idea/modules.xml Mon May 23 16:53:35 2011
@@ -4,20 +4,15 @@
<modules>
<module filepath="$PROJECT_DIR$/parent.iml" />
<module filepath="$PROJECT_DIR$/lucene/lucene.iml" />
- <module filepath="$PROJECT_DIR$/lucene/contrib/ant/ant.iml" />
- <module filepath="$PROJECT_DIR$/lucene/contrib/db/bdb/bdb.iml" />
- <module filepath="$PROJECT_DIR$/lucene/contrib/db/bdb-je/bdb-je.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/demo/demo.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/highlighter/highlighter.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/instantiated/instantiated.iml" />
- <module filepath="$PROJECT_DIR$/lucene/contrib/lucli/lucli.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/memory/memory.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/misc/misc.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/queries/queries.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/queryparser/queryparser.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/spatial/spatial.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/spellchecker/spellchecker.iml" />
- <module filepath="$PROJECT_DIR$/lucene/contrib/swing/swing.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/wordnet/wordnet.iml" />
<module filepath="$PROJECT_DIR$/lucene/contrib/xml-query-parser/xml-query-parser.iml" />
<module filepath="$PROJECT_DIR$/modules/analysis/common/common.iml" />
Modified: lucene/dev/branches/flexscoring/dev-tools/idea/.idea/workspace.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/dev-tools/idea/.idea/workspace.xml?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/dev-tools/idea/.idea/workspace.xml (original)
+++ lucene/dev/branches/flexscoring/dev-tools/idea/.idea/workspace.xml Mon May 23 16:53:35 2011
@@ -8,27 +8,6 @@
<option name="VM_PARAMETERS" value="-ea -Dtests.luceneMatchVersion=4.0 -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
- <configuration default="false" name="ant contrib" type="JUnit" factoryName="JUnit">
- <module name="ant" />
- <option name="TEST_OBJECT" value="package" />
- <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/lucene/build/contrib/ant" />
- <option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
- <option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
- </configuration>
- <configuration default="false" name="bdb contrib" type="JUnit" factoryName="JUnit">
- <module name="bdb" />
- <option name="TEST_OBJECT" value="package" />
- <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/lucene/build/contrib/db" />
- <option name="VM_PARAMETERS" value="-ea -DtempDir=bdb-temp" />
- <option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
- </configuration>
- <configuration default="false" name="bdb-je contrib" type="JUnit" factoryName="JUnit">
- <module name="bdb-je" />
- <option name="TEST_OBJECT" value="package" />
- <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/lucene/build/contrib/db" />
- <option name="VM_PARAMETERS" value="-ea -DtempDir=bdb-je-temp" />
- <option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
- </configuration>
<configuration default="false" name="benchmark module" type="JUnit" factoryName="JUnit">
<module name="benchmark" />
<option name="TEST_OBJECT" value="package" />
@@ -106,13 +85,6 @@
<option name="VM_PARAMETERS" value="-ea -Dlucene.version=4.0-SNAPSHOT -DtempDir=temp -Dtests.linedocsfile=europarl.lines.txt.gz" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
- <configuration default="false" name="lucli contrib" type="JUnit" factoryName="JUnit">
- <module name="lucli" />
- <option name="TEST_OBJECT" value="package" />
- <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/lucene/build/contrib/lucli" />
- <option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
- <option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
- </configuration>
<configuration default="false" name="memory contrib" type="JUnit" factoryName="JUnit">
<module name="memory" />
<option name="TEST_OBJECT" value="package" />
@@ -183,13 +155,6 @@
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
- <configuration default="false" name="swing contrib" type="JUnit" factoryName="JUnit">
- <module name="swing" />
- <option name="TEST_OBJECT" value="package" />
- <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/lucene/build/contrib/swing" />
- <option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
- <option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
- </configuration>
<configuration default="false" name="uima contrib" type="JUnit" factoryName="JUnit">
<module name="uima" />
<option name="TEST_OBJECT" value="package" />
@@ -211,37 +176,32 @@
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
- <list size="30">
+ <list size="25">
<item index="0" class="java.lang.String" itemvalue="JUnit.analysis-extras contrib" />
- <item index="1" class="java.lang.String" itemvalue="JUnit.ant contrib" />
- <item index="2" class="java.lang.String" itemvalue="JUnit.bdb contrib" />
- <item index="3" class="java.lang.String" itemvalue="JUnit.bdb-je contrib" />
- <item index="4" class="java.lang.String" itemvalue="JUnit.benchmark module" />
- <item index="5" class="java.lang.String" itemvalue="JUnit.clustering contrib" />
- <item index="6" class="java.lang.String" itemvalue="JUnit.common analysis module" />
- <item index="7" class="java.lang.String" itemvalue="JUnit.dataimporthandler contrib" />
- <item index="8" class="java.lang.String" itemvalue="JUnit.extraction contrib" />
- <item index="9" class="java.lang.String" itemvalue="JUnit.extras from dataimporthandler contrib" />
- <item index="10" class="java.lang.String" itemvalue="JUnit.grouping module" />
- <item index="11" class="java.lang.String" itemvalue="JUnit.highlighter contrib" />
- <item index="12" class="java.lang.String" itemvalue="JUnit.icu analysis module" />
- <item index="13" class="java.lang.String" itemvalue="JUnit.instantiated contrib" />
- <item index="14" class="java.lang.String" itemvalue="JUnit.lucene" />
- <item index="15" class="java.lang.String" itemvalue="JUnit.lucli contrib" />
- <item index="16" class="java.lang.String" itemvalue="JUnit.memory contrib" />
- <item index="17" class="java.lang.String" itemvalue="JUnit.misc contrib" />
- <item index="18" class="java.lang.String" itemvalue="JUnit.phonetic analysis module" />
- <item index="19" class="java.lang.String" itemvalue="JUnit.queries contrib" />
- <item index="20" class="java.lang.String" itemvalue="JUnit.queryparser contrib" />
- <item index="21" class="java.lang.String" itemvalue="JUnit.smartcn analysis module" />
- <item index="22" class="java.lang.String" itemvalue="JUnit.solr" />
- <item index="23" class="java.lang.String" itemvalue="JUnit.spatial contrib" />
- <item index="24" class="java.lang.String" itemvalue="JUnit.spellchecker contrib" />
- <item index="25" class="java.lang.String" itemvalue="JUnit.stempel analysis module" />
- <item index="26" class="java.lang.String" itemvalue="JUnit.swing contrib" />
- <item index="27" class="java.lang.String" itemvalue="JUnit.uima contrib" />
- <item index="28" class="java.lang.String" itemvalue="JUnit.wordnet contrib" />
- <item index="29" class="java.lang.String" itemvalue="JUnit.xml-query-parser contrib" />
+ <item index="1" class="java.lang.String" itemvalue="JUnit.benchmark module" />
+ <item index="2" class="java.lang.String" itemvalue="JUnit.clustering contrib" />
+ <item index="3" class="java.lang.String" itemvalue="JUnit.common analysis module" />
+ <item index="4" class="java.lang.String" itemvalue="JUnit.dataimporthandler contrib" />
+ <item index="5" class="java.lang.String" itemvalue="JUnit.extraction contrib" />
+ <item index="6" class="java.lang.String" itemvalue="JUnit.extras from dataimporthandler contrib" />
+ <item index="7" class="java.lang.String" itemvalue="JUnit.grouping module" />
+ <item index="8" class="java.lang.String" itemvalue="JUnit.highlighter contrib" />
+ <item index="9" class="java.lang.String" itemvalue="JUnit.icu analysis module" />
+ <item index="10" class="java.lang.String" itemvalue="JUnit.instantiated contrib" />
+ <item index="11" class="java.lang.String" itemvalue="JUnit.lucene" />
+ <item index="12" class="java.lang.String" itemvalue="JUnit.memory contrib" />
+ <item index="13" class="java.lang.String" itemvalue="JUnit.misc contrib" />
+ <item index="14" class="java.lang.String" itemvalue="JUnit.phonetic analysis module" />
+ <item index="15" class="java.lang.String" itemvalue="JUnit.queries contrib" />
+ <item index="16" class="java.lang.String" itemvalue="JUnit.queryparser contrib" />
+ <item index="17" class="java.lang.String" itemvalue="JUnit.smartcn analysis module" />
+ <item index="18" class="java.lang.String" itemvalue="JUnit.solr" />
+ <item index="19" class="java.lang.String" itemvalue="JUnit.spatial contrib" />
+ <item index="20" class="java.lang.String" itemvalue="JUnit.spellchecker contrib" />
+ <item index="21" class="java.lang.String" itemvalue="JUnit.stempel analysis module" />
+ <item index="22" class="java.lang.String" itemvalue="JUnit.uima contrib" />
+ <item index="23" class="java.lang.String" itemvalue="JUnit.wordnet contrib" />
+ <item index="24" class="java.lang.String" itemvalue="JUnit.xml-query-parser contrib" />
</list>
</component>
</project>
Modified: lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/pom.xml.template?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/pom.xml.template (original)
+++ lucene/dev/branches/flexscoring/dev-tools/maven/lucene/contrib/pom.xml.template Mon May 23 16:53:35 2011
@@ -31,19 +31,15 @@
<name>Lucene Contrib aggregator POM</name>
<packaging>pom</packaging>
<modules>
- <module>ant</module>
- <module>db</module>
<module>demo</module>
<module>highlighter</module>
<module>instantiated</module>
- <module>lucli</module>
<module>memory</module>
<module>misc</module>
<module>queries</module>
<module>queryparser</module>
<module>spatial</module>
<module>spellchecker</module>
- <module>swing</module>
<module>wordnet</module>
<module>xml-query-parser</module>
</modules>
Modified: lucene/dev/branches/flexscoring/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/dev-tools/maven/pom.xml.template?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/flexscoring/dev-tools/maven/pom.xml.template Mon May 23 16:53:35 2011
@@ -131,16 +131,6 @@
<version>r05</version>
</dependency>
<dependency>
- <groupId>com.sleepycat</groupId>
- <artifactId>berkeleydb</artifactId>
- <version>4.7.25</version>
- </dependency>
- <dependency>
- <groupId>com.sleepycat</groupId>
- <artifactId>berkeleydb-je</artifactId>
- <version>3.3.93</version>
- </dependency>
- <dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.7.0</version>
@@ -206,11 +196,6 @@
<version>0.9.1</version>
</dependency>
<dependency>
- <groupId>jtidy</groupId>
- <artifactId>jtidy</artifactId>
- <version>4aug2000r7-dev</version>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
@@ -585,25 +570,6 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>get-jars-and-poms</id>
- <phase>install</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <ant dir="lucene/contrib/db/bdb" target="get-db-jar"/>
- <ant dir="lucene/contrib/db/bdb-je" target="get-je-jar"/>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<executions>
<execution>
@@ -635,34 +601,6 @@
</configuration>
</execution>
<execution>
- <id>install-berkeleydb</id>
- <phase>install</phase>
- <goals>
- <goal>install-file</goal>
- </goals>
- <configuration>
- <groupId>com.sleepycat</groupId>
- <artifactId>berkeleydb</artifactId>
- <version>4.7.25</version>
- <packaging>jar</packaging>
- <file>lucene/contrib/db/bdb/lib/db-4.7.25.jar</file>
- </configuration>
- </execution>
- <execution>
- <id>install-berkeleydb-je</id>
- <phase>install</phase>
- <goals>
- <goal>install-file</goal>
- </goals>
- <configuration>
- <groupId>com.sleepycat</groupId>
- <artifactId>berkeleydb-je</artifactId>
- <version>3.3.93</version>
- <packaging>jar</packaging>
- <file>lucene/contrib/db/bdb-je/lib/je-3.3.93.jar</file>
- </configuration>
- </execution>
- <execution>
<id>install-solr-commons-csv</id>
<phase>install</phase>
<goals>
Modified: lucene/dev/branches/flexscoring/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/CHANGES.txt?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/flexscoring/lucene/CHANGES.txt Mon May 23 16:53:35 2011
@@ -486,6 +486,10 @@ New features
document IDs and scores encountered during the search, and "replay" them to
another Collector. (Mike McCandless, Shai Erera)
+* LUCENE-3112: Added experimental IndexWriter.add/updateDocuments,
+ enabling a block of documents to be indexed, atomically, with
+ guaranteed sequential docIDs. (Mike McCandless)
+
API Changes
* LUCENE-3061: IndexWriter's getNextMerge() and merge(OneMerge) are now public
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Mon May 23 16:53:35 2011
@@ -45,6 +45,11 @@ import org.apache.lucene.store.FSDirecto
* @lucene.experimental You can easily
* accidentally remove segments from your index so be
* careful!
+ *
+ * <p><b>NOTE</b>: this tool is unaware of documents added
+ * atomically via {@link IndexWriter#addDocuments} or {@link
+ * IndexWriter#updateDocuments}, which means it can easily
+ * break up such document groups.
*/
public class IndexSplitter {
public SegmentInfos infos;
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Mon May 23 16:53:35 2011
@@ -40,6 +40,11 @@ import org.apache.lucene.util.Version;
* <p>Note 2: the disadvantage of this tool is that source index needs to be
* read as many times as there are parts to be created, hence the name of this
* tool.
+ *
+ * <p><b>NOTE</b>: this tool is unaware of documents added
+ * atomically via {@link IndexWriter#addDocuments} or {@link
+ * IndexWriter#updateDocuments}, which means it can easily
+ * break up such document groups.
*/
public class MultiPassIndexSplitter {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Mon May 23 16:53:35 2011
@@ -274,11 +274,9 @@ final class DocumentsWriter {
flushControl.setClosed();
}
- boolean updateDocument(final Document doc, final Analyzer analyzer,
- final Term delTerm) throws CorruptIndexException, IOException {
+ private boolean preUpdate() throws CorruptIndexException, IOException {
ensureOpen();
boolean maybeMerge = false;
- final boolean isUpdate = delTerm != null;
if (flushControl.anyStalledThreads() || flushControl.numQueuedFlushes() > 0) {
// Help out flushing any queued DWPTs so we can un-stall:
if (infoStream != null) {
@@ -303,13 +301,30 @@ final class DocumentsWriter {
message("continue indexing after helpling out flushing DocumentsWriter is healthy");
}
}
+ return maybeMerge;
+ }
+
+ private boolean postUpdate(DocumentsWriterPerThread flushingDWPT, boolean maybeMerge) throws IOException {
+ if (flushingDWPT != null) {
+ maybeMerge |= doFlush(flushingDWPT);
+ } else {
+ final DocumentsWriterPerThread nextPendingFlush = flushControl.nextPendingFlush();
+ if (nextPendingFlush != null) {
+ maybeMerge |= doFlush(nextPendingFlush);
+ }
+ }
- final ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(),
- this, doc);
+ return maybeMerge;
+ }
+
+ boolean updateDocuments(final Iterable<Document> docs, final Analyzer analyzer,
+ final Term delTerm) throws CorruptIndexException, IOException {
+ boolean maybeMerge = preUpdate();
+
+ final ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), this);
final DocumentsWriterPerThread flushingDWPT;
try {
-
if (!perThread.isActive()) {
ensureOpen();
assert false: "perThread is not active but we are still open";
@@ -317,27 +332,53 @@ final class DocumentsWriter {
final DocumentsWriterPerThread dwpt = perThread.perThread;
try {
- dwpt.updateDocument(doc, analyzer, delTerm);
- numDocsInRAM.incrementAndGet();
+ final int docCount = dwpt.updateDocuments(docs, analyzer, delTerm);
+ numDocsInRAM.addAndGet(docCount);
} finally {
if (dwpt.checkAndResetHasAborted()) {
flushControl.doOnAbort(perThread);
}
}
+ final boolean isUpdate = delTerm != null;
flushingDWPT = flushControl.doAfterDocument(perThread, isUpdate);
} finally {
perThread.unlock();
}
+
+ return postUpdate(flushingDWPT, maybeMerge);
+ }
+
+ boolean updateDocument(final Document doc, final Analyzer analyzer,
+ final Term delTerm) throws CorruptIndexException, IOException {
+
+ boolean maybeMerge = preUpdate();
+
+ final ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), this);
+ final DocumentsWriterPerThread flushingDWPT;
- if (flushingDWPT != null) {
- maybeMerge |= doFlush(flushingDWPT);
- } else {
- final DocumentsWriterPerThread nextPendingFlush = flushControl.nextPendingFlush();
- if (nextPendingFlush != null) {
- maybeMerge |= doFlush(nextPendingFlush);
+ try {
+
+ if (!perThread.isActive()) {
+ ensureOpen();
+ assert false: "perThread is not active but we are still open";
}
+
+ final DocumentsWriterPerThread dwpt = perThread.perThread;
+ try {
+ dwpt.updateDocument(doc, analyzer, delTerm);
+ numDocsInRAM.incrementAndGet();
+ } finally {
+ if (dwpt.checkAndResetHasAborted()) {
+ flushControl.doOnAbort(perThread);
+ }
+ }
+ final boolean isUpdate = delTerm != null;
+ flushingDWPT = flushControl.doAfterDocument(perThread, isUpdate);
+ } finally {
+ perThread.unlock();
}
- return maybeMerge;
+
+ return postUpdate(flushingDWPT, maybeMerge);
}
private boolean doFlush(DocumentsWriterPerThread flushingDWPT) throws IOException {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Mon May 23 16:53:35 2011
@@ -68,7 +68,7 @@ public final class DocumentsWriterFlushC
this.stallControl = new DocumentsWriterStallControl();
this.perThreadPool = documentsWriter.perThreadPool;
this.flushPolicy = documentsWriter.flushPolicy;
- this.hardMaxBytesPerDWPT = config.getRAMPerThreadHardLimitMB() * 1024 * 1024;;
+ this.hardMaxBytesPerDWPT = config.getRAMPerThreadHardLimitMB() * 1024 * 1024;
this.config = config;
this.documentsWriter = documentsWriter;
}
@@ -162,8 +162,6 @@ public final class DocumentsWriterFlushC
stallControl.updateStalled(this);
assert assertMemory();
}
-
-
}
synchronized void doAfterFlush(DocumentsWriterPerThread dwpt) {
@@ -217,7 +215,7 @@ public final class DocumentsWriterFlushC
assert assertMemory();
// Take it out of the loop this DWPT is stale
perThreadPool.replaceForFlush(state, closed);
- }finally {
+ } finally {
stallControl.updateStalled(this);
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Mon May 23 16:53:35 2011
@@ -104,7 +104,7 @@ public class DocumentsWriterPerThread {
// largish:
doc = null;
analyzer = null;
- }
+ }
}
static class FlushedSegment {
@@ -253,6 +253,82 @@ public class DocumentsWriterPerThread {
finishDocument(delTerm);
}
+ public int updateDocuments(Iterable<Document> docs, Analyzer analyzer, Term delTerm) throws IOException {
+ assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
+ assert deleteQueue != null;
+ docState.analyzer = analyzer;
+ if (segment == null) {
+ // this call is synchronized on IndexWriter.segmentInfos
+ segment = writer.newSegmentName();
+ assert numDocsInRAM == 0;
+ }
+
+ int docCount = 0;
+ try {
+ for(Document doc : docs) {
+ docState.doc = doc;
+ docState.docID = numDocsInRAM;
+ docCount++;
+
+ boolean success = false;
+ try {
+ consumer.processDocument(fieldInfos);
+ success = true;
+ } finally {
+ if (!success) {
+ // An exc is being thrown...
+
+ if (!aborting) {
+ // One of the documents hit a non-aborting
+ // exception (eg something happened during
+ // analysis). We now go and mark any docs
+ // from this batch that we had already indexed
+ // as deleted:
+ int docID = docState.docID;
+ final int endDocID = docID - docCount;
+ while (docID > endDocID) {
+ deleteDocID(docID);
+ docID--;
+ }
+
+ // Incr here because finishDocument will not
+ // be called (because an exc is being thrown):
+ numDocsInRAM++;
+ fieldInfos.revertUncommitted();
+ } else {
+ abort();
+ }
+ }
+ }
+ success = false;
+ try {
+ consumer.finishDocument();
+ success = true;
+ } finally {
+ if (!success) {
+ abort();
+ }
+ }
+
+ finishDocument(null);
+ }
+
+ // Apply delTerm only after all indexing has
+ // succeeded, but apply it only to docs prior to when
+ // this batch started:
+ if (delTerm != null) {
+ deleteQueue.add(delTerm, deleteSlice);
+ assert deleteSlice.isTailItem(delTerm) : "expected the delete term as the tail item";
+ deleteSlice.apply(pendingDeletes, numDocsInRAM-docCount);
+ }
+
+ } finally {
+ docState.clear();
+ }
+
+ return docCount;
+ }
+
private void finishDocument(Term delTerm) throws IOException {
/*
* here we actually finish the document in two steps 1. push the delete into
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Mon May 23 16:53:35 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.util.Iterator;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
import org.apache.lucene.index.codecs.CodecProvider;
@@ -212,7 +211,7 @@ public abstract class DocumentsWriterPer
// don't recycle DWPT by default
}
- public abstract ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter, Document doc);
+ public abstract ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter);
/**
* Returns an iterator providing access to all {@link ThreadState}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon May 23 16:53:35 2011
@@ -1228,6 +1228,111 @@ public class IndexWriter implements Clos
}
/**
+ * Atomically adds a block of documents with sequentially
+ * assigned document IDs, such that an external reader
+ * will see all or none of the documents.
+ *
+ * <p><b>WARNING</b>: the index does not currently record
+ * which documents were added as a block. Today this is
+ * fine, because merging will preserve the block (as long
+ * as none them were deleted). But it's possible in the
+ * future that Lucene may more aggressively re-order
+ * documents (for example, perhaps to obtain better index
+ * compression), in which case you may need to fully
+ * re-index your documents at that time.
+ *
+ * <p>See {@link #addDocument(Document)} for details on
+ * index and IndexWriter state after an Exception, and
+ * flushing/merging temporary free space requirements.</p>
+ *
+ * <p><b>NOTE</b>: tools that do offline splitting of an index
+ * (for example, IndexSplitter in contrib) or
+ * re-sorting of documents (for example, IndexSorter in
+ * contrib) are not aware of these atomically added documents
+ * and will likely break them up. Use such tools at your
+ * own risk!
+ *
+ * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
+ * you should immediately close the writer. See <a
+ * href="#OOME">above</a> for details.</p>
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ *
+ * @lucene.experimental
+ */
+ public void addDocuments(Iterable<Document> docs) throws CorruptIndexException, IOException {
+ addDocuments(docs, analyzer);
+ }
+
+ /**
+ * Atomically adds a block of documents, analyzed using the
+ * provided analyzer, with sequentially assigned document
+ * IDs, such that an external reader will see all or none
+ * of the documents.
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ *
+ * @lucene.experimental
+ */
+ public void addDocuments(Iterable<Document> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ updateDocuments(null, docs, analyzer);
+ }
+
+ /**
+ * Atomically deletes documents matching the provided
+ * delTerm and adds a block of documents with sequentially
+ * assigned document IDs, such that an external reader
+ * will see all or none of the documents.
+ *
+ * See {@link #addDocuments(Iterable)}.
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ *
+ * @lucene.experimental
+ */
+ public void updateDocuments(Term delTerm, Iterable<Document> docs) throws CorruptIndexException, IOException {
+ updateDocuments(delTerm, docs, analyzer);
+ }
+
+ /**
+ * Atomically deletes documents matching the provided
+ * delTerm and adds a block of documents, analyzed using
+ * the provided analyzer, with sequentially
+ * assigned document IDs, such that an external reader
+ * will see all or none of the documents.
+ *
+ * See {@link #addDocuments(Iterable)}.
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ *
+ * @lucene.experimental
+ */
+ public void updateDocuments(Term delTerm, Iterable<Document> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ ensureOpen();
+ try {
+ boolean success = false;
+ boolean anySegmentFlushed = false;
+ try {
+ anySegmentFlushed = docWriter.updateDocuments(docs, analyzer, delTerm);
+ success = true;
+ } finally {
+ if (!success && infoStream != null) {
+ message("hit exception updating document");
+ }
+ }
+ if (anySegmentFlushed) {
+ maybeMerge();
+ }
+ } catch (OutOfMemoryError oom) {
+ handleOOM(oom, "updateDocuments");
+ }
+ }
+
+ /**
* Deletes the document(s) containing <code>term</code>.
*
* <p><b>NOTE</b>: if this method hits an OutOfMemoryError
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon May 23 16:53:35 2011
@@ -29,7 +29,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -70,174 +69,7 @@ public class SegmentReader extends Index
IndexInput singleNormStream;
AtomicInteger singleNormRef;
- CoreReaders core;
-
- // Holds core readers that are shared (unchanged) when
- // SegmentReader is cloned or reopened
- static final class CoreReaders {
-
- // Counts how many other reader share the core objects
- // (freqStream, proxStream, tis, etc.) of this reader;
- // when coreRef drops to 0, these core objects may be
- // closed. A given instance of SegmentReader may be
- // closed, even those it shares core objects with other
- // SegmentReaders:
- private final AtomicInteger ref = new AtomicInteger(1);
-
- final String segment;
- final FieldInfos fieldInfos;
-
- final FieldsProducer fields;
-
- final Directory dir;
- final Directory cfsDir;
- final int readBufferSize;
- final int termsIndexDivisor;
-
- private final SegmentReader origInstance;
-
- FieldsReader fieldsReaderOrig;
- TermVectorsReader termVectorsReaderOrig;
- CompoundFileReader cfsReader;
- CompoundFileReader storeCFSReader;
-
- CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
-
- if (termsIndexDivisor == 0) {
- throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
- }
-
- segment = si.name;
- final SegmentCodecs segmentCodecs = si.getSegmentCodecs();
- this.readBufferSize = readBufferSize;
- this.dir = dir;
-
- boolean success = false;
-
- try {
- Directory dir0 = dir;
- if (si.getUseCompoundFile()) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
- dir0 = cfsReader;
- }
- cfsDir = dir0;
- si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos
- fieldInfos = si.getFieldInfos();
-
- this.termsIndexDivisor = termsIndexDivisor;
-
- // Ask codec for its Fields
- fields = segmentCodecs.codec().fieldsProducer(new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor));
- assert fields != null;
-
- success = true;
- } finally {
- if (!success) {
- decRef();
- }
- }
-
- // Must assign this at the end -- if we hit an
- // exception above core, we don't want to attempt to
- // purge the FieldCache (will hit NPE because core is
- // not assigned yet).
- this.origInstance = origInstance;
- }
-
- synchronized TermVectorsReader getTermVectorsReaderOrig() {
- return termVectorsReaderOrig;
- }
-
- synchronized FieldsReader getFieldsReaderOrig() {
- return fieldsReaderOrig;
- }
-
- synchronized void incRef() {
- ref.incrementAndGet();
- }
-
- synchronized Directory getCFSReader() {
- return cfsReader;
- }
-
- synchronized void decRef() throws IOException {
-
- if (ref.decrementAndGet() == 0) {
-
- if (fields != null) {
- fields.close();
- }
-
- if (termVectorsReaderOrig != null) {
- termVectorsReaderOrig.close();
- }
-
- if (fieldsReaderOrig != null) {
- fieldsReaderOrig.close();
- }
-
- if (cfsReader != null) {
- cfsReader.close();
- }
-
- if (storeCFSReader != null) {
- storeCFSReader.close();
- }
-
- // Now, notify any ReaderFinished listeners:
- if (origInstance != null) {
- origInstance.notifyReaderFinishedListeners();
- }
- }
- }
-
- synchronized void openDocStores(SegmentInfo si) throws IOException {
-
- assert si.name.equals(segment);
-
- if (fieldsReaderOrig == null) {
- final Directory storeDir;
- if (si.getDocStoreOffset() != -1) {
- if (si.getDocStoreIsCompoundFile()) {
- assert storeCFSReader == null;
- storeCFSReader = new CompoundFileReader(dir,
- IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
- readBufferSize);
- storeDir = storeCFSReader;
- assert storeDir != null;
- } else {
- storeDir = dir;
- assert storeDir != null;
- }
- } else if (si.getUseCompoundFile()) {
- // In some cases, we were originally opened when CFS
- // was not used, but then we are asked to open doc
- // stores after the segment has switched to CFS
- if (cfsReader == null) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
- }
- storeDir = cfsReader;
- assert storeDir != null;
- } else {
- storeDir = dir;
- assert storeDir != null;
- }
-
- final String storesSegment = si.getDocStoreSegment();
- fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize,
- si.getDocStoreOffset(), si.docCount);
-
- // Verify two sources of "maxDoc" agree:
- if (si.getDocStoreOffset() == -1 && fieldsReaderOrig.size() != si.docCount) {
- throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + fieldsReaderOrig.size() + " but segmentInfo shows " + si.docCount);
- }
-
- if (si.getHasVectors()) { // open term vector files only as needed
- termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount);
- }
- }
- }
- }
+ SegmentCoreReaders core;
/**
* Sets the initial value
@@ -248,8 +80,6 @@ public class SegmentReader extends Index
return (FieldsReader) core.getFieldsReaderOrig().clone();
}
}
-
-
Map<String,SegmentNorms> norms = new HashMap<String,SegmentNorms>();
@@ -281,7 +111,7 @@ public class SegmentReader extends Index
boolean success = false;
try {
- instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor);
+ instance.core = new SegmentCoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor);
if (doOpenStores) {
instance.core.openDocStores(si);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Mon May 23 16:53:35 2011
@@ -139,7 +139,7 @@ final class TermVectorsTermsWriter exten
}
}
- assert lastDocID == docState.docID;
+ assert lastDocID == docState.docID: "lastDocID=" + lastDocID + " docState.docID=" + docState.docID;
lastDocID++;
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java Mon May 23 16:53:35 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; //javadoc
/**
@@ -48,12 +47,10 @@ public class ThreadAffinityDocumentsWrit
}
@Override
- public ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter, Document doc) {
+ public ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter) {
ThreadState threadState = threadBindings.get(requestingThread);
- if (threadState != null) {
- if (threadState.tryLock()) {
- return threadState;
- }
+ if (threadState != null && threadState.tryLock()) {
+ return threadState;
}
ThreadState minThreadState = null;
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Mon May 23 16:53:35 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Iterator;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -97,8 +98,43 @@ public class RandomIndexWriter implement
* Adds a Document.
* @see IndexWriter#addDocument(Document)
*/
- public void addDocument(Document doc) throws IOException {
- w.addDocument(doc);
+ public void addDocument(final Document doc) throws IOException {
+ if (r.nextInt(5) == 3) {
+ // TODO: maybe, we should simply buffer up added docs
+ // (but we need to clone them), and only when
+ // getReader, commit, etc. are called, we do an
+ // addDocuments? Would be better testing.
+ w.addDocuments(new Iterable<Document>() {
+
+ // @Override -- not until Java 1.6
+ public Iterator<Document> iterator() {
+ return new Iterator<Document>() {
+ boolean done;
+
+ // @Override -- not until Java 1.6
+ public boolean hasNext() {
+ return !done;
+ }
+
+ // @Override -- not until Java 1.6
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override -- not until Java 1.6
+ public Document next() {
+ if (done) {
+ throw new IllegalStateException();
+ }
+ done = true;
+ return doc;
+ }
+ };
+ }
+ });
+ } else {
+ w.addDocument(doc);
+ }
maybeCommit();
}
@@ -116,12 +152,53 @@ public class RandomIndexWriter implement
}
}
+ public void addDocuments(Iterable<Document> docs) throws IOException {
+ w.addDocuments(docs);
+ maybeCommit();
+ }
+
+ public void updateDocuments(Term delTerm, Iterable<Document> docs) throws IOException {
+ w.updateDocuments(delTerm, docs);
+ maybeCommit();
+ }
+
/**
* Updates a document.
* @see IndexWriter#updateDocument(Term, Document)
*/
- public void updateDocument(Term t, Document doc) throws IOException {
- w.updateDocument(t, doc);
+ public void updateDocument(Term t, final Document doc) throws IOException {
+ if (r.nextInt(5) == 3) {
+ w.updateDocuments(t, new Iterable<Document>() {
+
+ // @Override -- not until Java 1.6
+ public Iterator<Document> iterator() {
+ return new Iterator<Document>() {
+ boolean done;
+
+ // @Override -- not until Java 1.6
+ public boolean hasNext() {
+ return !done;
+ }
+
+ // @Override -- not until Java 1.6
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override -- not until Java 1.6
+ public Document next() {
+ if (done) {
+ throw new IllegalStateException();
+ }
+ done = true;
+ return doc;
+ }
+ };
+ }
+ });
+ } else {
+ w.updateDocument(t, doc);
+ }
maybeCommit();
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon May 23 16:53:35 2011
@@ -17,24 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
@@ -43,9 +35,54 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestIndexWriterExceptions extends LuceneTestCase {
+ private static class DocCopyIterator implements Iterable<Document> {
+ private final Document doc;
+ private final int count;
+
+ public DocCopyIterator(Document doc, int count) {
+ this.count = count;
+ this.doc = doc;
+ }
+
+ // @Override -- not until Java 1.6
+ public Iterator<Document> iterator() {
+ return new Iterator<Document>() {
+ int upto;
+
+ // @Override -- not until Java 1.6
+ public boolean hasNext() {
+ return upto < count;
+ }
+
+ // @Override -- not until Java 1.6
+ public Document next() {
+ upto++;
+ return doc;
+ }
+
+ // @Override -- not until Java 1.6
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }
+
private class IndexerThread extends Thread {
IndexWriter writer;
@@ -87,7 +124,11 @@ public class TestIndexWriterExceptions e
idField.setValue(id);
Term idTerm = new Term("id", id);
try {
- writer.updateDocument(idTerm, doc);
+ if (r.nextBoolean()) {
+ writer.updateDocuments(idTerm, new DocCopyIterator(doc, _TestUtil.nextInt(r, 1, 20)));
+ } else {
+ writer.updateDocument(idTerm, doc);
+ }
} catch (RuntimeException re) {
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": EXC: ");
@@ -136,7 +177,7 @@ public class TestIndexWriterExceptions e
@Override
boolean testPoint(String name) {
- if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(20) == 17) {
+ if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
new Throwable().printStackTrace(System.out);
@@ -267,6 +308,8 @@ public class TestIndexWriterExceptions e
}
}
+ private static String CRASH_FAIL_MESSAGE = "I'm experiencing problems";
+
private class CrashingFilter extends TokenFilter {
String fieldName;
int count;
@@ -279,7 +322,7 @@ public class TestIndexWriterExceptions e
@Override
public boolean incrementToken() throws IOException {
if (this.fieldName.equals("crash") && count++ >= 4)
- throw new IOException("I'm experiencing problems");
+ throw new IOException(CRASH_FAIL_MESSAGE);
return input.incrementToken();
}
@@ -1278,4 +1321,141 @@ public class TestIndexWriterExceptions e
}
}
}
+
+ public void testAddDocsNonAbortingException() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int numDocs1 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs1;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final List<Document> docs = new ArrayList<Document>();
+ for(int docCount=0;docCount<7;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ if (docCount == 4) {
+ Field f = newField("crash", "", Field.Index.ANALYZED);
+ doc.add(f);
+ MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ }
+ }
+ try {
+ w.addDocuments(docs);
+ // BUG: CrashingFilter didn't
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
+ }
+
+ final int numDocs2 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs2;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final IndexSearcher s = new IndexSearcher(r);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("content", "silly"));
+ pq.add(new Term("content", "content"));
+ assertEquals(0, s.search(pq, 1).totalHits);
+
+ pq = new PhraseQuery();
+ pq.add(new Term("content", "good"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs1+numDocs2, s.search(pq, 1).totalHits);
+ r.close();
+ dir.close();
+ }
+
+
+ public void testUpdateDocsNonAbortingException() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int numDocs1 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs1;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ // Use addDocs (no exception) to get docs in the index:
+ final List<Document> docs = new ArrayList<Document>();
+ final int numDocs2 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs2;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("subid", "subs", Field.Index.NOT_ANALYZED));
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ }
+ w.addDocuments(docs);
+
+ final int numDocs3 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs3;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ docs.clear();
+ final int limit = _TestUtil.nextInt(random, 2, 25);
+ final int crashAt = random.nextInt(limit);
+ for(int docCount=0;docCount<limit;docCount++) {
+ Document doc = new Document();
+ docs.add(doc);
+ doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
+ doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
+ if (docCount == crashAt) {
+ Field f = newField("crash", "", Field.Index.ANALYZED);
+ doc.add(f);
+ MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ f.setTokenStream(new CrashingFilter("crash", tokenizer));
+ }
+ }
+
+ try {
+ w.updateDocuments(new Term("subid", "subs"), docs);
+ // BUG: CrashingFilter didn't
+ fail("did not hit expected exception");
+ } catch (IOException ioe) {
+ // expected
+ assertEquals(CRASH_FAIL_MESSAGE, ioe.getMessage());
+ }
+
+ final int numDocs4 = random.nextInt(25);
+ for(int docCount=0;docCount<numDocs4;docCount++) {
+ Document doc = new Document();
+ doc.add(newField("content", "good content", Field.Index.ANALYZED));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final IndexSearcher s = new IndexSearcher(r);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("content", "silly"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs2, s.search(pq, 1).totalHits);
+
+ pq = new PhraseQuery();
+ pq.add(new Term("content", "good"));
+ pq.add(new Term("content", "content"));
+ assertEquals(numDocs1+numDocs3+numDocs4, s.search(pq, 1).totalHits);
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Mon May 23 16:53:35 2011
@@ -21,33 +21,35 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.HashSet;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.document.Field;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
import org.junit.Test;
@@ -57,6 +59,39 @@ import org.junit.Test;
public class TestNRTThreads extends LuceneTestCase {
+ private static class SubDocs {
+ public final String packID;
+ public final List<String> subIDs;
+ public boolean deleted;
+
+ public SubDocs(String packID, List<String> subIDs) {
+ this.packID = packID;
+ this.subIDs = subIDs;
+ }
+ }
+
+ // TODO: is there a pre-existing way to do this!!!
+ private Document cloneDoc(Document doc1) {
+ final Document doc2 = new Document();
+ for(Fieldable f : doc1.getFields()) {
+ Field field1 = (Field) f;
+
+ Field field2 = new Field(field1.name(),
+ field1.stringValue(),
+ field1.isStored() ? Field.Store.YES : Field.Store.NO,
+ field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
+ if (field1.getOmitNorms()) {
+ field2.setOmitNorms(true);
+ }
+ if (field1.getOmitTermFreqAndPositions()) {
+ field2.setOmitTermFreqAndPositions(true);
+ }
+ doc2.add(field2);
+ }
+
+ return doc2;
+ }
+
@Test
public void testNRTThreads() throws Exception {
@@ -121,13 +156,16 @@ public class TestNRTThreads extends Luce
final int NUM_INDEX_THREADS = 2;
final int NUM_SEARCH_THREADS = 3;
+
final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : 5;
final AtomicBoolean failed = new AtomicBoolean();
final AtomicInteger addCount = new AtomicInteger();
final AtomicInteger delCount = new AtomicInteger();
+ final AtomicInteger packCount = new AtomicInteger();
final Set<String> delIDs = Collections.synchronizedSet(new HashSet<String>());
+ final List<SubDocs> allSubDocs = Collections.synchronizedList(new ArrayList<SubDocs>());
final long stopTime = System.currentTimeMillis() + RUN_TIME_SEC*1000;
Thread[] threads = new Thread[NUM_INDEX_THREADS];
@@ -135,7 +173,9 @@ public class TestNRTThreads extends Luce
threads[thread] = new Thread() {
@Override
public void run() {
+ // TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works:
final List<String> toDeleteIDs = new ArrayList<String>();
+ final List<SubDocs> toDeleteSubDocs = new ArrayList<SubDocs>();
while(System.currentTimeMillis() < stopTime && !failed.get()) {
try {
Document doc = docs.nextDoc();
@@ -153,7 +193,92 @@ public class TestNRTThreads extends Luce
if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": add doc id:" + doc.get("docid"));
}
- writer.addDocument(doc);
+
+ if (random.nextBoolean()) {
+ // Add a pack of adjacent sub-docs
+ final String packID;
+ final SubDocs delSubDocs;
+ if (toDeleteSubDocs.size() > 0 && random.nextBoolean()) {
+ delSubDocs = toDeleteSubDocs.get(random.nextInt(toDeleteSubDocs.size()));
+ assert !delSubDocs.deleted;
+ toDeleteSubDocs.remove(delSubDocs);
+ // reuse prior packID
+ packID = delSubDocs.packID;
+ } else {
+ delSubDocs = null;
+ // make new packID
+ packID = packCount.getAndIncrement() + "";
+ }
+
+ final Field packIDField = newField("packID", packID, Field.Store.YES, Field.Index.NOT_ANALYZED);
+ final List<String> docIDs = new ArrayList<String>();
+ final SubDocs subDocs = new SubDocs(packID, docIDs);
+ final List<Document> docsList = new ArrayList<Document>();
+
+ allSubDocs.add(subDocs);
+ doc.add(packIDField);
+ docsList.add(cloneDoc(doc));
+ docIDs.add(doc.get("docid"));
+
+ final int maxDocCount = _TestUtil.nextInt(random, 1, 10);
+ while(docsList.size() < maxDocCount) {
+ doc = docs.nextDoc();
+ if (doc == null) {
+ break;
+ }
+ docsList.add(cloneDoc(doc));
+ docIDs.add(doc.get("docid"));
+ }
+ addCount.addAndGet(docsList.size());
+
+ if (delSubDocs != null) {
+ delSubDocs.deleted = true;
+ delIDs.addAll(delSubDocs.subIDs);
+ delCount.addAndGet(delSubDocs.subIDs.size());
+ if (VERBOSE) {
+ System.out.println("TEST: update pack packID=" + delSubDocs.packID + " count=" + docsList.size() + " docs=" + docIDs);
+ }
+ writer.updateDocuments(new Term("packID", delSubDocs.packID), docsList);
+ /*
+ // non-atomic:
+ writer.deleteDocuments(new Term("packID", delSubDocs.packID));
+ for(Document subDoc : docsList) {
+ writer.addDocument(subDoc);
+ }
+ */
+ } else {
+ if (VERBOSE) {
+ System.out.println("TEST: add pack packID=" + packID + " count=" + docsList.size() + " docs=" + docIDs);
+ }
+ writer.addDocuments(docsList);
+
+ /*
+ // non-atomic:
+ for(Document subDoc : docsList) {
+ writer.addDocument(subDoc);
+ }
+ */
+ }
+ doc.removeField("packID");
+
+ if (random.nextInt(5) == 2) {
+ if (VERBOSE) {
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + packID);
+ }
+ toDeleteSubDocs.add(subDocs);
+ }
+
+ } else {
+ writer.addDocument(doc);
+ addCount.getAndIncrement();
+
+ if (random.nextInt(5) == 3) {
+ if (VERBOSE) {
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
+ }
+ toDeleteIDs.add(doc.get("docid"));
+ }
+ }
} else {
// we use update but it never replaces a
// prior doc
@@ -161,14 +286,17 @@ public class TestNRTThreads extends Luce
//System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("docid"));
}
writer.updateDocument(new Term("docid", doc.get("docid")), doc);
- }
- if (random.nextInt(5) == 3) {
- if (VERBOSE) {
- //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
+ addCount.getAndIncrement();
+
+ if (random.nextInt(5) == 3) {
+ if (VERBOSE) {
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
+ }
+ toDeleteIDs.add(doc.get("docid"));
}
- toDeleteIDs.add(doc.get("docid"));
}
- if (random.nextInt(50) == 17) {
+
+ if (random.nextInt(30) == 17) {
if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": apply " + toDeleteIDs.size() + " deletes");
}
@@ -184,8 +312,19 @@ public class TestNRTThreads extends Luce
}
delIDs.addAll(toDeleteIDs);
toDeleteIDs.clear();
+
+ for(SubDocs subDocs : toDeleteSubDocs) {
+ assert !subDocs.deleted;
+ writer.deleteDocuments(new Term("packID", subDocs.packID));
+ subDocs.deleted = true;
+ if (VERBOSE) {
+ System.out.println(" del subs: " + subDocs.subIDs + " packID=" + subDocs.packID);
+ }
+ delIDs.addAll(subDocs.subIDs);
+ delCount.addAndGet(subDocs.subIDs.size());
+ }
+ toDeleteSubDocs.clear();
}
- addCount.getAndIncrement();
if (addedField != null) {
doc.removeField(addedField);
}
@@ -356,7 +495,7 @@ public class TestNRTThreads extends Luce
if (VERBOSE) {
System.out.println("TEST: done join [" + (System.currentTimeMillis()-t0) + " ms]; addCount=" + addCount + " delCount=" + delCount);
}
-
+
final IndexReader r2 = writer.getReader();
final IndexSearcher s = newSearcher(r2);
boolean doFail = false;
@@ -367,6 +506,43 @@ public class TestNRTThreads extends Luce
doFail = true;
}
}
+
+ // Make sure each group of sub-docs are still in docID order:
+ for(SubDocs subDocs : allSubDocs) {
+ if (!subDocs.deleted) {
+ // We sort by relevance but the scores should be identical so sort falls back to by docID:
+ TopDocs hits = s.search(new TermQuery(new Term("packID", subDocs.packID)), 20);
+ assertEquals(subDocs.subIDs.size(), hits.totalHits);
+ int lastDocID = -1;
+ int startDocID = -1;
+ for(ScoreDoc scoreDoc : hits.scoreDocs) {
+ final int docID = scoreDoc.doc;
+ if (lastDocID != -1) {
+ assertEquals(1+lastDocID, docID);
+ } else {
+ startDocID = docID;
+ }
+ lastDocID = docID;
+ final Document doc = s.doc(docID);
+ assertEquals(subDocs.packID, doc.get("packID"));
+ }
+
+ lastDocID = startDocID - 1;
+ for(String subID : subDocs.subIDs) {
+ hits = s.search(new TermQuery(new Term("docid", subID)), 1);
+ assertEquals(1, hits.totalHits);
+ final int docID = hits.scoreDocs[0].doc;
+ if (lastDocID != -1) {
+ assertEquals(1+lastDocID, docID);
+ }
+ lastDocID = docID;
+ }
+ } else {
+ for(String subID : subDocs.subIDs) {
+ assertEquals(0, s.search(new TermQuery(new Term("docid", subID)), 1).totalHits);
+ }
+ }
+ }
final int endID = Integer.parseInt(docs.nextDoc().get("docid"));
for(int id=0;id<endID;id++) {
Modified: lucene/dev/branches/flexscoring/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/CHANGES.txt?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/CHANGES.txt (original)
+++ lucene/dev/branches/flexscoring/solr/CHANGES.txt Mon May 23 16:53:35 2011
@@ -333,6 +333,10 @@ Bug Fixes
in strings since those characters are not valid in javascript strings
(although they are valid in JSON strings). (yonik)
+* SOLR-2536: Add ReloadCacheRequestHandler to fix ExternalFileField bug (if reopenReaders
+ set to true and no index segments have been changed, commit cannot trigger reload
+ external file). (koji)
+
Other Changes
----------------------
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/FileFloatSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/FileFloatSource.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/FileFloatSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/FileFloatSource.java Mon May 23 16:53:35 2011
@@ -16,23 +16,38 @@
*/
package org.apache.solr.search.function;
-import org.apache.lucene.index.IndexReader;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.BytesRef;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.schema.SchemaField;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.handler.RequestHandlerUtils;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.QParser;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.VersionedFile;
-
-import java.io.*;
-import java.util.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Obtains float field values from an external file.
@@ -99,6 +114,10 @@ public class FileFloatSource extends Val
+ ",defVal="+defVal+",dataDir="+dataDir+")";
}
+
+ public static void resetCache(){
+ floatCache.resetCache();
+ }
private final float[] getCachedFloats(IndexReader reader) {
return (float[])floatCache.get(reader, new Entry(this));
@@ -150,6 +169,14 @@ public class FileFloatSource extends Val
return value;
}
+
+ public void resetCache(){
+ synchronized(readerCache){
+ // Map.clear() is optional and can throw UnsipportedOperationException,
+ // but readerCache is WeakHashMap and it supports clear().
+ readerCache.clear();
+ }
+ }
}
static Object onlyForTesting; // set to the last value
@@ -272,5 +299,44 @@ public class FileFloatSource extends Val
return vals;
}
+ public static class ReloadCacheRequestHandler extends RequestHandlerBase {
+
+ static final Logger log = LoggerFactory.getLogger(ReloadCacheRequestHandler.class);
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+ throws Exception {
+ FileFloatSource.resetCache();
+ log.debug("readerCache has been reset.");
+
+ UpdateRequestProcessor processor =
+ req.getCore().getUpdateProcessingChain(null).createProcessor(req, rsp);
+ try{
+ RequestHandlerUtils.handleCommit(req, processor, req.getParams(), true);
+ }
+ finally{
+ processor.finish();
+ }
+ }
+
+ @Override
+ public String getDescription() {
+ return "Reload readerCache request handler";
+ }
+
+ @Override
+ public String getSource() {
+ return "$URL$";
+ }
+
+ @Override
+ public String getSourceId() {
+ return "$Id$";
+ }
+
+ @Override
+ public String getVersion() {
+ return "$Revision$";
+ }
+ }
}
Modified: lucene/dev/branches/flexscoring/solr/src/test-files/solr/conf/solrconfig-functionquery.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/test-files/solr/conf/solrconfig-functionquery.xml?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/test-files/solr/conf/solrconfig-functionquery.xml (original)
+++ lucene/dev/branches/flexscoring/solr/src/test-files/solr/conf/solrconfig-functionquery.xml Mon May 23 16:53:35 2011
@@ -30,6 +30,9 @@
<updateHandler class="solr.DirectUpdateHandler2"/>
<requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
+ <requestHandler name="/reloadCache"
+ class="org.apache.solr.search.function.FileFloatSource$ReloadCacheRequestHandler" />
+
<!-- test ValueSourceParser plugins -->
<valueSourceParser name="nvl" class="org.apache.solr.search.function.NvlValueSourceParser">
<float name="nvlFloatValue">0.0</float>
Modified: lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1126579&r1=1126578&r2=1126579&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java Mon May 23 16:53:35 2011
@@ -22,6 +22,8 @@ import org.apache.lucene.search.DefaultS
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Ignore;
@@ -191,7 +193,7 @@ public class TestFunctionQuery extends S
}
@Test
- public void testExternalField() {
+ public void testExternalField() throws Exception {
String field = "foo_extf";
float[] ids = {100,-4,0,10,25,5,77,23,55,-78,-45,-24,63,78,94,22,34,54321,261,-627};
@@ -210,8 +212,7 @@ public class TestFunctionQuery extends S
assertTrue(orig == FileFloatSource.onlyForTesting);
makeExternalFile(field, "0=1","UTF-8");
- assertU(adoc("id", "10000")); // will get same reader if no index change
- assertU(commit());
+ assertU(h.query("/reloadCache",lrf.makeRequest("","")));
singleTest(field, "sqrt(\0)");
assertTrue(orig != FileFloatSource.onlyForTesting);
@@ -247,8 +248,7 @@ public class TestFunctionQuery extends S
makeExternalFile(field, sb.toString(),"UTF-8");
// make it visible
- assertU(adoc("id", "10001")); // will get same reader if no index change
- assertU(commit());
+ assertU(h.query("/reloadCache",lrf.makeRequest("","")));
// test it
float[] answers = new float[ids.length*2];