You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/01/04 15:53:21 UTC
svn commit: r1649347 [3/31] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ dev-tools/idea/solr/contrib/dataimporthandler-extras/
dev-tools/idea/solr/contrib/extraction/
dev-tools/idea/solr/contrib/map-reduce/
dev-tools/idea/solr/contrib/velocity/ ...
Modified: lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java (original)
+++ lucene/dev/branches/lucene6005/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java Sun Jan 4 14:53:12 2015
@@ -50,7 +50,7 @@ import org.apache.lucene.util.BytesRef;
public class CachingNaiveBayesClassifier extends SimpleNaiveBayesClassifier {
//for caching classes this will be the classification class list
private ArrayList<BytesRef> cclasses = new ArrayList<>();
- // its a term-inmap style map, where the inmap contains class-hit pairs to the
+ // it's a term-inmap style map, where the inmap contains class-hit pairs to the
// upper term
private Map<String, Map<BytesRef, Integer>> termCClassHitCache = new HashMap<>();
// the term frequency in classes
@@ -185,7 +185,7 @@ public class CachingNaiveBayesClassifier
Map<BytesRef, Integer> searched = new ConcurrentHashMap<>();
- // if we dont get the answer, but its relevant we must search it and insert to the cache
+ // if we dont get the answer, but it's relevant we must search it and insert to the cache
if (insertPoint != null || !justCachedTerms) {
for (BytesRef cclass : cclasses) {
BooleanQuery booleanQuery = new BooleanQuery();
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Sun Jan 4 14:53:12 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.blockte
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -854,7 +855,7 @@ public class BlockTermsReader extends Fi
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
if (indexReader != null) {
resources.add(Accountables.namedAccountable("term index", indexReader));
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java Sun Jan 4 14:53:12 2015
@@ -17,11 +17,19 @@ package org.apache.lucene.codecs.blockte
* limitations under the License.
*/
-import org.apache.lucene.store.IndexInput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.BytesRef;
@@ -29,14 +37,6 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.io.IOException;
-
-import org.apache.lucene.index.IndexFileNames;
-
/**
* TermsIndexReader for simple every Nth terms indexes.
*
@@ -262,7 +262,7 @@ public class FixedGapTermsIndexReader ex
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
if (termOffsets != null) {
resources.add(Accountables.namedAccountable("term lengths", termOffsets));
@@ -303,7 +303,7 @@ public class FixedGapTermsIndexReader ex
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
return Accountables.namedAccountables("field", fields);
}
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blockte
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -165,7 +166,7 @@ public class VariableGapTermsIndexReader
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (fst == null) {
return Collections.emptyList();
} else {
@@ -208,7 +209,7 @@ public class VariableGapTermsIndexReader
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
return Accountables.namedAccountables("field", fields);
}
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java Sun Jan 4 14:53:12 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.blocktr
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -29,9 +30,9 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.store.IndexInput;
@@ -234,7 +235,7 @@ public final class OrdsBlockTreeTermsRea
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("field", fields));
resources.add(Accountables.namedAccountable("delegate", postingsReader));
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java Sun Jan 4 14:53:12 2015
@@ -900,9 +900,15 @@ public final class OrdsBlockTreeTermsWri
private final RAMOutputStream bytesWriter = new RAMOutputStream();
}
+ private boolean closed;
+
@Override
public void close() throws IOException {
-
+ if (closed) {
+ return;
+ }
+ closed = true;
+
boolean success = false;
try {
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blocktr
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
@@ -175,7 +176,7 @@ final class OrdsFieldReader extends Term
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (index == null) {
return Collections.emptyList();
} else {
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Sun Jan 4 14:53:12 2015
@@ -19,12 +19,13 @@ package org.apache.lucene.codecs.bloom;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.Map.Entry;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
@@ -407,7 +408,7 @@ public final class BloomFilteringPosting
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("field", bloomsByFieldName));
if (delegateFieldsProducer != null) {
@@ -483,9 +484,14 @@ public final class BloomFilteringPosting
}
}
+ private boolean closed;
+
@Override
public void close() throws IOException {
-
+ if (closed) {
+ return;
+ }
+ closed = true;
delegateFieldsConsumer.close();
// Now we are done accumulating values for these fields
@@ -499,9 +505,7 @@ public final class BloomFilteringPosting
}
String bloomFileName = IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
- IndexOutput bloomOutput = null;
- try {
- bloomOutput = state.directory.createOutput(bloomFileName, state.context);
+ try (IndexOutput bloomOutput = state.directory.createOutput(bloomFileName, state.context)) {
CodecUtil.writeIndexHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// remember the name of the postings format we will delegate to
bloomOutput.writeString(delegatePostingsFormat.getName());
@@ -515,8 +519,6 @@ public final class BloomFilteringPosting
saveAppropriatelySizedBloomFilter(bloomOutput, bloomFilter, fieldInfo);
}
CodecUtil.writeFooter(bloomOutput);
- } finally {
- IOUtils.close(bloomOutput);
}
//We are done with large bitsets so no need to keep them hanging around
bloomFilters.clear();
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java Sun Jan 4 14:53:12 2015
@@ -262,7 +262,7 @@ class DirectDocValuesConsumer extends Do
}
}
- // note: this might not be the most efficient... but its fairly simple
+ // note: this might not be the most efficient... but it's fairly simple
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
meta.writeVInt(field.number);
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java Sun Jan 4 14:53:12 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.memory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -259,7 +260,7 @@ class DirectDocValuesProducer extends Do
}
@Override
- public synchronized Iterable<? extends Accountable> getChildResources() {
+ public synchronized Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("numeric field", numericInstances));
resources.addAll(Accountables.namedAccountables("binary field", binaryInstances));
@@ -661,7 +662,7 @@ class DirectDocValuesProducer extends Do
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
if (address != null) {
resources.add(Accountables.namedAccountable("addresses", RamUsageEstimator.sizeOf(address)));
@@ -700,7 +701,7 @@ class DirectDocValuesProducer extends Do
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
return docToOrd.getChildResources();
}
@@ -724,7 +725,7 @@ class DirectDocValuesProducer extends Do
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
if (docToAddress != null) {
resources.add(Accountables.namedAccountable("addresses", docToAddress));
@@ -753,7 +754,7 @@ class DirectDocValuesProducer extends Do
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
if (docToOrdAddress != null) {
resources.add(Accountables.namedAccountable("addresses", docToOrdAddress));
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.memory;
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
@@ -162,7 +163,7 @@ public final class DirectPostingsFormat
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
return Accountables.namedAccountables("field", fields);
}
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Sun Jan 4 14:53:12 2015
@@ -18,21 +18,26 @@ package org.apache.lucene.codecs.memory;
*/
import java.io.IOException;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.TreeMap;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
@@ -41,26 +46,22 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.automaton.ByteRunAutomaton;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.CodecUtil;
/**
* FST-based terms dictionary reader.
@@ -292,7 +293,7 @@ public class FSTOrdTermsReader extends F
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (index == null) {
return Collections.emptyList();
} else {
@@ -869,7 +870,7 @@ public class FSTOrdTermsReader extends F
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("field", fields));
resources.add(Accountables.namedAccountable("delegate", postingsReader));
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Sun Jan 4 14:53:12 2015
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.memory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -32,10 +33,10 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
@@ -44,8 +45,8 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Accountables;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -191,7 +192,7 @@ public class FSTTermsReader extends Fiel
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (dict == null) {
return Collections.emptyList();
} else {
@@ -767,7 +768,7 @@ public class FSTTermsReader extends Fiel
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("field", fields));
resources.add(Accountables.namedAccountable("delegate", postingsReader));
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Sun Jan 4 14:53:12 2015
@@ -196,7 +196,7 @@ class MemoryDocValuesConsumer extends Do
int numBlocks = maxDoc / BLOCK_SIZE;
float avgBPV = blockSum / (float)numBlocks;
// just a heuristic, with tiny amounts of blocks our estimate is skewed as we ignore the final "incomplete" block.
- // with at least 4 blocks its pretty accurate. The difference must also be significant (according to acceptable overhead).
+ // with at least 4 blocks it's pretty accurate. The difference must also be significant (according to acceptable overhead).
if (numBlocks >= 4 && (avgBPV+avgBPV*acceptableOverheadRatio) < deltaBPV.bitsPerValue) {
doBlock = true;
}
@@ -335,7 +335,7 @@ class MemoryDocValuesConsumer extends Do
meta.writeVInt(minLength);
meta.writeVInt(maxLength);
- // if minLength == maxLength, its a fixed-length byte[], we are done (the addresses are implicit)
+ // if minLength == maxLength, it's a fixed-length byte[], we are done (the addresses are implicit)
// otherwise, we need to record the length fields...
if (minLength != maxLength) {
meta.writeVInt(PackedInts.VERSION_CURRENT);
@@ -432,7 +432,7 @@ class MemoryDocValuesConsumer extends Do
}
}
- // note: this might not be the most efficient... but its fairly simple
+ // note: this might not be the most efficient... but it's fairly simple
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
meta.writeVInt(field.number);
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java Sun Jan 4 14:53:12 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.memory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -296,7 +297,7 @@ class MemoryDocValuesProducer extends Do
}
@Override
- public synchronized Iterable<? extends Accountable> getChildResources() {
+ public synchronized Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("numeric field", numericInfo));
resources.addAll(Accountables.namedAccountables("pagedbytes field", pagedBytesInstances));
@@ -799,7 +800,7 @@ class MemoryDocValuesProducer extends Do
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
if (addresses != null) {
resources.add(Accountables.namedAccountable("addresses", addresses));
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.memory;
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
@@ -33,10 +34,10 @@ import org.apache.lucene.index.CorruptIn
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
@@ -404,14 +405,19 @@ public final class MemoryPostingsFormat
}
}
+ private boolean closed;
+
@Override
public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+
// EOF marker:
- try {
+ try (IndexOutput out = this.out) {
out.writeVInt(0);
CodecUtil.writeFooter(out);
- } finally {
- out.close();
}
}
}
@@ -955,7 +961,7 @@ public final class MemoryPostingsFormat
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (fst == null) {
return Collections.emptyList();
} else {
@@ -1030,7 +1036,7 @@ public final class MemoryPostingsFormat
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
return Accountables.namedAccountables("field", fields);
}
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java Sun Jan 4 14:53:12 2015
@@ -111,11 +111,11 @@ import org.apache.lucene.index.SegmentWr
* </pre>
* so the "ord section" begins at startOffset + (9+pattern.length+maxlength)*numValues.
* a document's ord list can be retrieved by seeking to "ord section" + (1+ordpattern.length())*docid
- * this is a comma-separated list, and its padded with spaces to be fixed width. so trim() and split() it.
+ * this is a comma-separated list, and it's padded with spaces to be fixed width. so trim() and split() it.
* and beware the empty string!
* an ord's value can be retrieved by seeking to startOffset + (9+pattern.length+maxlength)*ord
*
- * for sorted numerics, its encoded (not very creatively) as a comma-separated list of strings the same as binary.
+ * for sorted numerics, it's encoded (not very creatively) as a comma-separated list of strings the same as binary.
* beware the empty string!
*
* the reader can just scan this file when it opens, skipping over the data blocks
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Sun Jan 4 14:53:12 2015
@@ -180,7 +180,7 @@ class SimpleTextDocValuesReader extends
} catch (ParseException pe) {
throw new CorruptIndexException("failed to parse BigDecimal value", in, pe);
}
- SimpleTextUtil.readLine(in, scratch); // read the line telling us if its real or not
+ SimpleTextUtil.readLine(in, scratch); // read the line telling us if it's real or not
return BigInteger.valueOf(field.minValue).add(bd.toBigIntegerExact()).longValue();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Sun Jan 4 14:53:12 2015
@@ -17,8 +17,19 @@ package org.apache.lucene.codecs.simplet
* limitations under the License.
*/
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.DOC;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END_OFFSET;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.FIELD;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.FREQ;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.PAYLOAD;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.POS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.START_OFFSET;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.TERM;
+
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -57,16 +68,6 @@ import org.apache.lucene.util.fst.PairOu
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.DOC;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END_OFFSET;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.FIELD;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.FREQ;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.PAYLOAD;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.POS;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.START_OFFSET;
-import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.TERM;
-
class SimpleTextFieldsReader extends FieldsProducer {
private static final long BASE_RAM_BYTES_USED =
@@ -595,7 +596,7 @@ class SimpleTextFieldsReader extends Fie
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (fst == null) {
return Collections.emptyList();
} else {
@@ -700,7 +701,7 @@ class SimpleTextFieldsReader extends Fie
}
@Override
- public synchronized Iterable<? extends Accountable> getChildResources() {
+ public synchronized Collection<Accountable> getChildResources() {
return Accountables.namedAccountables("field", termsCache);
}
Modified: lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java Sun Jan 4 14:53:12 2015
@@ -18,9 +18,8 @@ package org.apache.lucene.codecs.simplet
*/
import java.io.IOException;
+import java.util.Collection;
-import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
@@ -82,7 +81,7 @@ public class SimpleTextNormsFormat exten
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
return impl.getChildResources();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java Sun Jan 4 14:53:12 2015
@@ -88,7 +88,7 @@ public abstract class Analyzer implement
/**
* Expert: create a new Analyzer with a custom {@link ReuseStrategy}.
* <p>
- * NOTE: if you just want to reuse on a per-field basis, its easier to
+ * NOTE: if you just want to reuse on a per-field basis, it's easier to
* use a subclass of {@link AnalyzerWrapper} such as
* <a href="{@docRoot}/../analyzers-common/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.html">
* PerFieldAnalyerWrapper</a> instead.
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/CachingTokenFilter.java Sun Jan 4 14:53:12 2015
@@ -28,11 +28,11 @@ import org.apache.lucene.util.AttributeS
* This class can be used if the token attributes of a TokenStream
* are intended to be consumed more than once. It caches
* all token attribute states locally in a List when the first call to
- * {@link #incrementToken()} is called.
- *
- * <P>CachingTokenFilter implements the optional method
- * {@link TokenStream#reset()}, which repositions the
- * stream to the first Token.
+ * {@link #incrementToken()} is called. Subsequent calls will used the cache.
+ * <p/>
+ * <em>Important:</em> Like any proper TokenFilter, {@link #reset()} propagates
+ * to the input, although only before {@link #incrementToken()} is called the
+ * first time. Prior to Lucene 5, it was never propagated.
*/
public final class CachingTokenFilter extends TokenFilter {
private List<AttributeSource.State> cache = null;
@@ -40,17 +40,31 @@ public final class CachingTokenFilter ex
private AttributeSource.State finalState;
/**
- * Create a new CachingTokenFilter around <code>input</code>,
- * caching its token attributes, which can be replayed again
- * after a call to {@link #reset()}.
+ * Create a new CachingTokenFilter around <code>input</code>. As with
+ * any normal TokenFilter, do <em>not</em> call reset on the input; this filter
+ * will do it normally.
*/
public CachingTokenFilter(TokenStream input) {
super(input);
}
-
+
+ /**
+ * Propagates reset if incrementToken has not yet been called. Otherwise
+ * it rewinds the iterator to the beginning of the cached list.
+ */
+ @Override
+ public void reset() throws IOException {
+ if (cache == null) {//first time
+ input.reset();
+ } else {
+ iterator = cache.iterator();
+ }
+ }
+
+ /** The first time called, it'll read and cache all tokens from the input. */
@Override
public final boolean incrementToken() throws IOException {
- if (cache == null) {
+ if (cache == null) {//first-time
// fill cache lazily
cache = new ArrayList<>(64);
fillCache();
@@ -65,7 +79,7 @@ public final class CachingTokenFilter ex
restoreState(iterator.next());
return true;
}
-
+
@Override
public final void end() {
if (finalState != null) {
@@ -73,20 +87,6 @@ public final class CachingTokenFilter ex
}
}
- /**
- * Rewinds the iterator to the beginning of the cached list.
- * <p>
- * Note that this does not call reset() on the wrapped tokenstream ever, even
- * the first time. You should reset() the inner tokenstream before wrapping
- * it with CachingTokenFilter.
- */
- @Override
- public void reset() {
- if (cache != null) {
- iterator = cache.iterator();
- }
- }
-
private void fillCache() throws IOException {
while (input.incrementToken()) {
cache.add(captureState());
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java Sun Jan 4 14:53:12 2015
@@ -90,7 +90,7 @@ public abstract class Tokenizer extends
throw new IllegalStateException("TokenStream contract violation: close() call missing");
}
this.inputPending = input;
- assert setReaderTestPoint();
+ setReaderTestPoint();
}
@Override
@@ -100,10 +100,8 @@ public abstract class Tokenizer extends
inputPending = ILLEGAL_STATE_READER;
}
- // only used by assert, for testing
- boolean setReaderTestPoint() {
- return true;
- }
+ // only used for testing
+ void setReaderTestPoint() {}
private static final Reader ILLEGAL_STATE_READER = new Reader() {
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java Sun Jan 4 14:53:12 2015
@@ -29,7 +29,7 @@ import org.apache.lucene.util.BytesRef;
* in the {@link org.apache.lucene.search.payloads} and
* {@link org.apache.lucene.search.spans} packages.
* <p>
- * NOTE: because the payload will be stored at each position, its usually
+ * NOTE: because the payload will be stored at each position, it's usually
* best to use the minimum number of bytes necessary. Some codec implementations
* may optimize payload storage when all payloads have the same length.
*
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Sun Jan 4 14:53:12 2015
@@ -18,14 +18,18 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
+import java.io.Reader;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
/**
* Codec API for writing stored fields:
@@ -81,6 +85,7 @@ public abstract class StoredFieldsWriter
for (int i=0;i<mergeState.storedFieldsReaders.length;i++) {
StoredFieldsReader storedFieldsReader = mergeState.storedFieldsReaders[i];
storedFieldsReader.checkIntegrity();
+ MergeVisitor visitor = new MergeVisitor(mergeState, i);
int maxDoc = mergeState.maxDocs[i];
Bits liveDocs = mergeState.liveDocs[i];
for (int docID=0;docID<maxDoc;docID++) {
@@ -88,16 +93,9 @@ public abstract class StoredFieldsWriter
// skip deleted docs
continue;
}
- // TODO: this could be more efficient using
- // FieldVisitor instead of loading/writing entire
- // doc; ie we just have to renumber the field number
- // on the fly?
- // NOTE: it's very important to first assign to doc then pass it to
- // fieldsWriter.addDocument; see LUCENE-1282
- DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(null);
+ startDocument();
storedFieldsReader.visitDocument(docID, visitor);
- Document doc = visitor.getDocument();
- addDocument(doc, mergeState.mergeFieldInfos);
+ finishDocument();
docCount++;
mergeState.checkAbort.work(300);
}
@@ -105,16 +103,137 @@ public abstract class StoredFieldsWriter
finish(mergeState.mergeFieldInfos, docCount);
return docCount;
}
+
+ final static IndexableFieldType STORED_TYPE = new IndexableFieldType() {
+ @Override
+ public boolean stored() {
+ return true;
+ }
+ };
- /** sugar method for startDocument() + writeField() for every stored field in the document */
- protected final void addDocument(Iterable<? extends IndexableField> doc, FieldInfos fieldInfos) throws IOException {
- startDocument();
+ /**
+ * A visitor that adds every field it sees.
+ * <p>
+ * Use like this:
+ * <pre>
+ * MergeVisitor visitor = new MergeVisitor(mergeState, readerIndex);
+ * for (...) {
+ * startDocument();
+ * storedFieldsReader.visitDocument(docID, visitor);
+ * finishDocument();
+ * }
+ * </pre>
+ */
+ protected class MergeVisitor extends StoredFieldVisitor implements IndexableField {
+ BytesRef binaryValue;
+ String stringValue;
+ Number numericValue;
+ FieldInfo currentField;
+ FieldInfos remapper;
+
+ /**
+ * Create new merge visitor.
+ */
+ public MergeVisitor(MergeState mergeState, int readerIndex) {
+ // if field numbers are aligned, we can save hash lookups
+ // on every field access. Otherwise, we need to lookup
+ // fieldname each time, and remap to a new number.
+ for (FieldInfo fi : mergeState.fieldInfos[readerIndex]) {
+ FieldInfo other = mergeState.mergeFieldInfos.fieldInfo(fi.number);
+ if (other == null || !other.name.equals(fi.name)) {
+ remapper = mergeState.mergeFieldInfos;
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+ reset(fieldInfo);
+ binaryValue = new BytesRef(value);
+ write();
+ }
+
+ @Override
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ reset(fieldInfo);
+ stringValue = value;
+ write();
+ }
- for (IndexableField field : doc) {
- writeField(fieldInfos.fieldInfo(field.name()), field);
+ @Override
+ public void intField(FieldInfo fieldInfo, int value) throws IOException {
+ reset(fieldInfo);
+ numericValue = value;
+ write();
}
- finishDocument();
+ @Override
+ public void longField(FieldInfo fieldInfo, long value) throws IOException {
+ reset(fieldInfo);
+ numericValue = value;
+ write();
+ }
+
+ @Override
+ public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+ reset(fieldInfo);
+ numericValue = value;
+ write();
+ }
+
+ @Override
+ public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ reset(fieldInfo);
+ numericValue = value;
+ write();
+ }
+
+ @Override
+ public Status needsField(FieldInfo fieldInfo) throws IOException {
+ return Status.YES;
+ }
+
+ @Override
+ public String name() {
+ return currentField.name;
+ }
+
+ @Override
+ public IndexableFieldType fieldType() {
+ return STORED_TYPE;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return binaryValue;
+ }
+
+ @Override
+ public String stringValue() {
+ return stringValue;
+ }
+
+ @Override
+ public Number numericValue() {
+ return numericValue;
+ }
+
+ void reset(FieldInfo field) {
+ if (remapper != null) {
+ // field numbers are not aligned, we need to remap to the new field number
+ currentField = remapper.fieldInfo(field.name);
+ } else {
+ currentField = field;
+ }
+ binaryValue = null;
+ stringValue = null;
+ numericValue = null;
+ }
+
+ void write() throws IOException {
+ writeField(currentField, this);
+ }
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Sun Jan 4 14:53:12 2015
@@ -116,7 +116,7 @@ public abstract class TermVectorsWriter
* @lucene.internal
*/
// TODO: we should probably nuke this and make a more efficient 4.x format
- // PreFlex-RW could then be slow and buffer (its only used in tests...)
+ // PreFlex-RW could then be slow and buffer (it's only used in tests...)
public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
int position = 0;
int lastOffset = 0;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java Sun Jan 4 14:53:12 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.blocktr
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -48,7 +49,7 @@ import org.apache.lucene.util.fst.Output
* approach is that seekExact is often able to
* determine a term cannot exist without doing any IO, and
* intersection with Automata is very fast. Note that this
- * terms dictionary has it's own fixed terms index (ie, it
+ * terms dictionary has its own fixed terms index (ie, it
* does not support a pluggable terms index
* implementation).
*
@@ -284,7 +285,7 @@ public final class BlockTreeTermsReader
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
resources.addAll(Accountables.namedAccountables("field", fields));
resources.add(Accountables.namedAccountable("delegate", postingsReader));
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java Sun Jan 4 14:53:12 2015
@@ -1138,9 +1138,15 @@ public final class BlockTreeTermsWriter
private final RAMOutputStream bytesWriter = new RAMOutputStream();
}
+ private boolean closed;
+
@Override
public void close() throws IOException {
-
+ if (closed) {
+ return;
+ }
+ closed = true;
+
boolean success = false;
try {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blocktr
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import org.apache.lucene.index.FieldInfo;
@@ -190,7 +191,7 @@ public final class FieldReader extends T
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
if (index == null) {
return Collections.emptyList();
} else {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java Sun Jan 4 14:53:12 2015
@@ -49,15 +49,16 @@ public class CompressingStoredFieldsForm
private final CompressionMode compressionMode;
private final int chunkSize;
private final int maxDocsPerChunk;
+ private final int blockSize;
/**
* Create a new {@link CompressingStoredFieldsFormat} with an empty segment
* suffix.
*
- * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int, int)
+ * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int, int, int)
*/
- public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) {
- this(formatName, "", compressionMode, chunkSize, maxDocsPerChunk);
+ public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk, int blockSize) {
+ this(formatName, "", compressionMode, chunkSize, maxDocsPerChunk, blockSize);
}
/**
@@ -92,10 +93,11 @@ public class CompressingStoredFieldsForm
* @param compressionMode the {@link CompressionMode} to use
* @param chunkSize the minimum number of bytes of a single chunk of stored documents
* @param maxDocsPerChunk the maximum number of documents in a single chunk
+ * @param blockSize the number of chunks to store in an index block
* @see CompressionMode
*/
public CompressingStoredFieldsFormat(String formatName, String segmentSuffix,
- CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk) {
+ CompressionMode compressionMode, int chunkSize, int maxDocsPerChunk, int blockSize) {
this.formatName = formatName;
this.segmentSuffix = segmentSuffix;
this.compressionMode = compressionMode;
@@ -107,6 +109,10 @@ public class CompressingStoredFieldsForm
throw new IllegalArgumentException("maxDocsPerChunk must be >= 1");
}
this.maxDocsPerChunk = maxDocsPerChunk;
+ if (blockSize < 1) {
+ throw new IllegalArgumentException("blockSize must be >= 1");
+ }
+ this.blockSize = blockSize;
}
@Override
@@ -120,13 +126,13 @@ public class CompressingStoredFieldsForm
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
IOContext context) throws IOException {
return new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context,
- formatName, compressionMode, chunkSize, maxDocsPerChunk);
+ formatName, compressionMode, chunkSize, maxDocsPerChunk, blockSize);
}
@Override
public String toString() {
return getClass().getSimpleName() + "(compressionMode=" + compressionMode
- + ", chunkSize=" + chunkSize + ", maxDocsPerChunk=" + maxDocsPerChunk + ")";
+ + ", chunkSize=" + chunkSize + ", maxDocsPerChunk=" + maxDocsPerChunk + ", blockSize=" + blockSize + ")";
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java Sun Jan 4 14:53:12 2015
@@ -22,6 +22,8 @@ import static org.apache.lucene.util.Bit
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.index.CorruptIndexException;
@@ -188,7 +190,7 @@ public final class CompressingStoredFiel
}
@Override
- public Iterable<? extends Accountable> getChildResources() {
+ public Collection<Accountable> getChildResources() {
List<Accountable> resources = new ArrayList<>();
long docBaseDeltaBytes = RamUsageEstimator.shallowSizeOf(docBasesDeltas);
@@ -203,7 +205,7 @@ public final class CompressingStoredFiel
}
resources.add(Accountables.namedAccountable("start pointer deltas", startPointerDeltaBytes));
- return resources;
+ return Collections.unmodifiableList(resources);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexWriter.java Sun Jan 4 14:53:12 2015
@@ -72,9 +72,8 @@ import org.apache.lucene.util.packed.Pac
*/
public final class CompressingStoredFieldsIndexWriter implements Closeable {
- static final int BLOCK_SIZE = 1024; // number of chunks to serialize at once
-
final IndexOutput fieldsIndexOut;
+ final int blockSize;
int totalDocs;
int blockDocs;
int blockChunks;
@@ -83,12 +82,16 @@ public final class CompressingStoredFiel
final int[] docBaseDeltas;
final long[] startPointerDeltas;
- CompressingStoredFieldsIndexWriter(IndexOutput indexOutput) throws IOException {
+ CompressingStoredFieldsIndexWriter(IndexOutput indexOutput, int blockSize) throws IOException {
+ if (blockSize <= 0) {
+ throw new IllegalArgumentException("blockSize must be positive");
+ }
+ this.blockSize = blockSize;
this.fieldsIndexOut = indexOutput;
reset();
totalDocs = 0;
- docBaseDeltas = new int[BLOCK_SIZE];
- startPointerDeltas = new long[BLOCK_SIZE];
+ docBaseDeltas = new int[blockSize];
+ startPointerDeltas = new long[blockSize];
fieldsIndexOut.writeVInt(PackedInts.VERSION_CURRENT);
}
@@ -171,7 +174,7 @@ public final class CompressingStoredFiel
}
void writeIndex(int numDocs, long startPointer) throws IOException {
- if (blockChunks == BLOCK_SIZE) {
+ if (blockChunks == blockSize) {
writeBlock();
reset();
}