You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2010/11/19 19:30:09 UTC
svn commit: r1036971 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/ lucene/backwards/src/test/org/apache/lucene/analysis/
lucene/backwards/src/test/org/apache/lucene/document/
lucene/backwards/src/test/org/apache/lucene/index/ lu...
Author: uschindler
Date: Fri Nov 19 18:30:06 2010
New Revision: 1036971
URL: http://svn.apache.org/viewvc?rev=1036971&view=rev
Log:
LUCENE-2770: Make SegmentMerger always work on atomic subreaders, even when IndexWriter.addIndexes(IndexReader...) is used with DirectoryReaders or other MultiReaders. This saves lots of memory during merge of norms
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
lucene/dev/branches/branch_3x/lucene/build.xml (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishMinimalStemmer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishPossessiveFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/util/StemmerUtil.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/tartarus/snowball/TestApp.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/stempel/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/lib/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/src/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/BaseCharFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/LetterTokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/UAX29Tokenizer.jflex (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/StringHelper.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/LuceneResourcesWikiPage.html (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/LuceneResourcesWikiPageURLs.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/email.addresses.from.random.text.with.email.addresses.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/random.text.with.email.addresses.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/random.text.with.urls.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/urls.from.random.text.with.urls.txt (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/collation/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/example/ (props changed)
lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar (props changed)
lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar (props changed)
lucene/dev/branches/branch_3x/solr/src/ (props changed)
lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common/ (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/ShingleFilterFactory.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/SynonymFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/SynonymMap.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/analysis/WordDelimiterIterator.java (props changed)
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/response/ (props changed)
lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template (props changed)
lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template (props changed)
lucene/dev/branches/branch_3x/solr/src/solrj/org/ (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestSynonymFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/analysis/TestTrimFilter.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/request/SimpleFacetsTest.java (props changed)
lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/update/AutoCommitTest.java (props changed)
lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/ (props changed)
lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Fri Nov 19 18:30:06 2010
@@ -520,6 +520,11 @@ Optimizations
* LUCENE-2760: Optimize SpanFirstQuery and SpanPositionRangeQuery.
(Robert Muir)
+* LUCENE-2770: Make SegmentMerger always work on atomic subreaders,
+ even when IndexWriter.addIndexes(IndexReader...) is used with
+ DirectoryReaders or other MultiReaders. This saves lots of memory
+ during merge of norms. (Uwe Schindler, Mike McCandless)
+
Build
* LUCENE-2124: Moved the JDK-based collation support from contrib/collation
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDoc.java Fri Nov 19 18:30:06 2010
@@ -185,7 +185,8 @@ public class TestDoc extends LuceneTestC
merger.add(r1);
merger.add(r2);
merger.merge();
- merger.closeReaders();
+ r1.close();
+ r2.close();
if (useCompoundFile) {
List filesToDelete = merger.createCompoundFile(merged + ".cfs");
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java Fri Nov 19 18:30:06 2010
@@ -67,7 +67,6 @@ public class TestSegmentMerger extends L
merger.add(reader1);
merger.add(reader2);
int docsMerged = merger.merge();
- merger.closeReaders();
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = SegmentReader.get(true, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java Fri Nov 19 18:30:06 2010
@@ -183,6 +183,11 @@ public class IndexSorter {
}
}
+ @Override
+ public IndexReader[] getSequentialSubReaders() {
+ return null;
+ }
+
public Document document(int n) throws IOException {
return document(n, null);
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Fri Nov 19 18:30:06 2010
@@ -221,6 +221,11 @@ public class MultiPassIndexSplitter {
}
@Override
+ public IndexReader[] getSequentialSubReaders() {
+ return null;
+ }
+
+ @Override
public TermPositions termPositions() throws IOException {
return new FilterTermPositions(in.termPositions()) {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Nov 19 18:30:06 2010
@@ -30,14 +30,12 @@ import org.apache.lucene.index.PayloadPr
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ReaderUtil;
/**
* The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add},
* into a single Segment. After adding the appropriate readers, call the merge method to combine the
* segments.
- *<P>
- * If the compoundFile flag is set, then the segments will be merged into a compound file.
- *
*
* @see #merge
* @see #add
@@ -113,16 +111,7 @@ final class SegmentMerger {
* @param reader
*/
final void add(IndexReader reader) {
- readers.add(reader);
- }
-
- /**
- *
- * @param i The index of the reader to return
- * @return The ith reader to be merged
- */
- final IndexReader segmentReader(int i) {
- return readers.get(i);
+ ReaderUtil.gatherSubReaders(readers, reader);
}
/**
@@ -166,17 +155,6 @@ final class SegmentMerger {
return mergedDocs;
}
- /**
- * close all IndexReaders that have been added.
- * Should not be called before merge().
- * @throws IOException
- */
- final void closeReaders() throws IOException {
- for (final IndexReader reader : readers) {
- reader.close();
- }
- }
-
final List<String> createCompoundFile(String fileName)
throws IOException {
CompoundFileWriter cfsWriter =
@@ -223,7 +201,7 @@ final class SegmentMerger {
return files;
}
- private void addIndexed(IndexReader reader, FieldInfos fInfos,
+ private static void addIndexed(IndexReader reader, FieldInfos fInfos,
Collection<String> names, boolean storeTermVectors,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
boolean storePayloads, boolean omitTFAndPositions)
@@ -456,7 +434,7 @@ final class SegmentMerger {
final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++];
TermVectorsReader matchingVectorsReader = null;
if (matchingSegmentReader != null) {
- TermVectorsReader vectorsReader = matchingSegmentReader.getTermVectorsReaderOrig();
+ TermVectorsReader vectorsReader = matchingSegmentReader.getTermVectorsReader();
// If the TV* files are an older format then they cannot read raw docs:
if (vectorsReader != null && vectorsReader.canReadRawDocs()) {
@@ -731,7 +709,7 @@ final class SegmentMerger {
output = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.NORMS_EXTENSION));
output.writeBytes(NORMS_HEADER,NORMS_HEADER.length);
}
- for ( IndexReader reader : readers) {
+ for (IndexReader reader : readers) {
int maxDoc = reader.maxDoc();
if (normBuffer == null || normBuffer.length < maxDoc) {
// the buffer is too small for the current segment
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java Fri Nov 19 18:30:06 2010
@@ -193,7 +193,8 @@ public class TestDoc extends LuceneTestC
merger.add(r1);
merger.add(r2);
merger.merge();
- merger.closeReaders();
+ r1.close();
+ r2.close();
if (useCompoundFile) {
List<String> filesToDelete = merger.createCompoundFile(merged + ".cfs");
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1036971&r1=1036970&r2=1036971&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Fri Nov 19 18:30:06 2010
@@ -74,7 +74,6 @@ public class TestSegmentMerger extends L
merger.add(reader1);
merger.add(reader2);
int docsMerged = merger.merge();
- merger.closeReaders();
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = SegmentReader.get(true, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);