You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [6/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/m...
Modified: lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/ClassificationResult.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/ClassificationResult.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/ClassificationResult.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/ClassificationResult.java Fri Jan 18 18:30:54 2013
@@ -17,29 +17,29 @@
package org.apache.lucene.classification;
/**
- * The result of a call to {@link Classifier#assignClass(String)} holding an assigned class and a score.
+ * The result of a call to {@link Classifier#assignClass(String)} holding an assigned class of type <code>T</code> and a score.
* @lucene.experimental
*/
-public class ClassificationResult {
+public class ClassificationResult<T> {
- private String assignedClass;
+ private T assignedClass;
private double score;
/**
* Constructor
- * @param assignedClass the class <code>String</code> assigned by a {@link Classifier}
+ * @param assignedClass the class <code>T</code> assigned by a {@link Classifier}
* @param score the score for the assignedClass as a <code>double</code>
*/
- public ClassificationResult(String assignedClass, double score) {
+ public ClassificationResult(T assignedClass, double score) {
this.assignedClass = assignedClass;
this.score = score;
}
/**
* retrieve the result class
- * @return a <code>String</code> representing an assigned class
+ * @return a <code>T</code> representing an assigned class
*/
- public String getAssignedClass() {
+ public T getAssignedClass() {
return assignedClass;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/Classifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/Classifier.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/Classifier.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/Classifier.java Fri Jan 18 18:30:54 2013
@@ -22,18 +22,19 @@ import org.apache.lucene.index.AtomicRea
import java.io.IOException;
/**
- * A classifier, see <code>http://en.wikipedia.org/wiki/Classifier_(mathematics)</code>
+ * A classifier, see <code>http://en.wikipedia.org/wiki/Classifier_(mathematics)</code>, which assign classes of type
+ * <code>T</code>
* @lucene.experimental
*/
-public interface Classifier {
+public interface Classifier<T> {
/**
* Assign a class (with score) to the given text String
* @param text a String containing text to be classified
- * @return a {@link ClassificationResult} holding assigned class and score
+ * @return a {@link ClassificationResult} holding assigned class of type <code>T</code> and score
* @throws IOException If there is a low-level I/O error.
*/
- public ClassificationResult assignClass(String text) throws IOException;
+ public ClassificationResult<T> assignClass(String text) throws IOException;
/**
* Train the classifier using the underlying Lucene index
Modified: lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java Fri Jan 18 18:30:54 2013
@@ -23,6 +23,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.io.StringReader;
@@ -35,7 +36,7 @@ import java.util.Map;
*
* @lucene.experimental
*/
-public class KNearestNeighborClassifier implements Classifier {
+public class KNearestNeighborClassifier implements Classifier<BytesRef> {
private MoreLikeThis mlt;
private String textFieldName;
@@ -52,35 +53,46 @@ public class KNearestNeighborClassifier
this.k = k;
}
+ /**
+ * {@inheritDoc}
+ */
@Override
- public ClassificationResult assignClass(String text) throws IOException {
+ public ClassificationResult<BytesRef> assignClass(String text) throws IOException {
Query q = mlt.like(new StringReader(text), textFieldName);
- TopDocs docs = indexSearcher.search(q, k);
+ TopDocs topDocs = indexSearcher.search(q, k);
+ return selectClassFromNeighbors(topDocs);
+ }
+ private ClassificationResult<BytesRef> selectClassFromNeighbors(TopDocs topDocs) throws IOException {
// TODO : improve the nearest neighbor selection
- Map<String, Integer> classCounts = new HashMap<String, Integer>();
- for (ScoreDoc scoreDoc : docs.scoreDocs) {
- String cl = indexSearcher.doc(scoreDoc.doc).getField(classFieldName).stringValue();
- Integer count = classCounts.get(cl);
- if (count != null) {
- classCounts.put(cl, count + 1);
- } else {
- classCounts.put(cl, 1);
+ Map<BytesRef, Integer> classCounts = new HashMap<BytesRef, Integer>();
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ BytesRef cl = new BytesRef(indexSearcher.doc(scoreDoc.doc).getField(classFieldName).stringValue());
+ if (cl != null) {
+ Integer count = classCounts.get(cl);
+ if (count != null) {
+ classCounts.put(cl, count + 1);
+ } else {
+ classCounts.put(cl, 1);
+ }
}
}
- int max = 0;
- String assignedClass = null;
- for (String cl : classCounts.keySet()) {
+ double max = 0;
+ BytesRef assignedClass = new BytesRef();
+ for (BytesRef cl : classCounts.keySet()) {
Integer count = classCounts.get(cl);
if (count > max) {
max = count;
- assignedClass = cl;
+ assignedClass = cl.clone();
}
}
- double score = max / k;
- return new ClassificationResult(assignedClass, score);
+ double score = max / (double) k;
+ return new ClassificationResult<BytesRef>(assignedClass, score);
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public void train(AtomicReader atomicReader, String textFieldName, String classFieldName, Analyzer analyzer) throws IOException {
this.textFieldName = textFieldName;
Modified: lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java Fri Jan 18 18:30:54 2013
@@ -38,9 +38,10 @@ import java.util.LinkedList;
/**
* A simplistic Lucene based NaiveBayes classifier, see <code>http://en.wikipedia.org/wiki/Naive_Bayes_classifier</code>
+ *
* @lucene.experimental
*/
-public class SimpleNaiveBayesClassifier implements Classifier {
+public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
private AtomicReader atomicReader;
private String textFieldName;
@@ -48,14 +49,19 @@ public class SimpleNaiveBayesClassifier
private int docsWithClassSize;
private Analyzer analyzer;
private IndexSearcher indexSearcher;
-
- /**
+
+ /**
* Creates a new NaiveBayes classifier.
* Note that you must call {@link #train(AtomicReader, String, String, Analyzer) train()} before you can
* classify any documents.
*/
- public SimpleNaiveBayesClassifier() {}
+ public SimpleNaiveBayesClassifier() {
+ }
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public void train(AtomicReader atomicReader, String textFieldName, String classFieldName, Analyzer analyzer)
throws IOException {
this.atomicReader = atomicReader;
@@ -79,32 +85,37 @@ public class SimpleNaiveBayesClassifier
return result.toArray(new String[result.size()]);
}
- public ClassificationResult assignClass(String inputDocument) throws IOException {
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ClassificationResult<BytesRef> assignClass(String inputDocument) throws IOException {
if (atomicReader == null) {
throw new RuntimeException("need to train the classifier first");
}
double max = 0d;
- String foundClass = null;
+ BytesRef foundClass = new BytesRef();
Terms terms = MultiFields.getTerms(atomicReader, classFieldName);
TermsEnum termsEnum = terms.iterator(null);
BytesRef next;
- while((next = termsEnum.next()) != null) {
+ String[] tokenizedDoc = tokenizeDoc(inputDocument);
+ while ((next = termsEnum.next()) != null) {
// TODO : turn it to be in log scale
- double clVal = calculatePrior(next) * calculateLikelihood(inputDocument, next);
+ double clVal = calculatePrior(next) * calculateLikelihood(tokenizedDoc, next);
if (clVal > max) {
max = clVal;
- foundClass = next.utf8ToString();
+ foundClass = next.clone();
}
}
- return new ClassificationResult(foundClass, max);
+ return new ClassificationResult<BytesRef>(foundClass, max);
}
- private double calculateLikelihood(String document, BytesRef c) throws IOException {
+ private double calculateLikelihood(String[] tokenizedDoc, BytesRef c) throws IOException {
// for each word
double result = 1d;
- for (String word : tokenizeDoc(document)) {
+ for (String word : tokenizedDoc) {
// search with text:word AND class:c
int hits = getWordFreqForClass(word, c);
Modified: lucene/dev/branches/LUCENE-2878/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java Fri Jan 18 18:30:54 2013
@@ -19,10 +19,12 @@ package org.apache.lucene.classification
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.After;
import org.junit.Before;
@@ -37,6 +39,7 @@ public abstract class ClassificationTest
private String classFieldName;
private Directory dir;
+ @Override
@Before
public void setUp() throws Exception {
super.setUp();
@@ -46,6 +49,7 @@ public abstract class ClassificationTest
classFieldName = "cat";
}
+ @Override
@After
public void tearDown() throws Exception {
super.tearDown();
@@ -53,15 +57,17 @@ public abstract class ClassificationTest
dir.close();
}
- protected void checkCorrectClassification(Classifier classifier, Analyzer analyzer) throws Exception {
+
+ protected void checkCorrectClassification(Classifier<BytesRef> classifier, Analyzer analyzer) throws Exception {
SlowCompositeReaderWrapper compositeReaderWrapper = null;
try {
populateIndex(analyzer);
compositeReaderWrapper = new SlowCompositeReaderWrapper(indexWriter.getReader());
classifier.train(compositeReaderWrapper, textFieldName, classFieldName, analyzer);
String newText = "Much is made of what the likes of Facebook, Google and Apple know about users. Truth is, Amazon may know more.";
- ClassificationResult classificationResult = classifier.assignClass(newText);
- assertEquals("technology", classificationResult.getAssignedClass());
+ ClassificationResult<BytesRef> classificationResult = classifier.assignClass(newText);
+ assertNotNull(classificationResult.getAssignedClass());
+ assertEquals(new BytesRef("technology"), classificationResult.getAssignedClass());
assertTrue(classificationResult.getScore() > 0);
} finally {
if (compositeReaderWrapper != null)
@@ -71,52 +77,57 @@ public abstract class ClassificationTest
private void populateIndex(Analyzer analyzer) throws Exception {
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(true);
+ ft.setStoreTermVectorOffsets(true);
+ ft.setStoreTermVectorPositions(true);
+
Document doc = new Document();
- doc.add(new TextField(textFieldName, "The traveling press secretary for Mitt Romney lost his cool and cursed at reporters " +
+ doc.add(new Field(textFieldName, "The traveling press secretary for Mitt Romney lost his cool and cursed at reporters " +
"who attempted to ask questions of the Republican presidential candidate in a public plaza near the Tomb of " +
- "the Unknown Soldier in Warsaw Tuesday.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ "the Unknown Soldier in Warsaw Tuesday.", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "Mitt Romney seeks to assure Israel and Iran, as well as Jewish voters in the United" +
- " States, that he will be tougher against Iran's nuclear ambitions than President Barack Obama.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ doc.add(new Field(textFieldName, "Mitt Romney seeks to assure Israel and Iran, as well as Jewish voters in the United" +
+ " States, that he will be tougher against Iran's nuclear ambitions than President Barack Obama.", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "And there's a threshold question that he has to answer for the American people and " +
+ doc.add(new Field(textFieldName, "And there's a threshold question that he has to answer for the American people and " +
"that's whether he is prepared to be commander-in-chief,\" she continued. \"As we look to the past events, we " +
- "know that this raises some questions about his preparedness and we'll see how the rest of his trip goes.\"", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ "know that this raises some questions about his preparedness and we'll see how the rest of his trip goes.\"", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "Still, when it comes to gun policy, many congressional Democrats have \"decided to " +
+ doc.add(new Field(textFieldName, "Still, when it comes to gun policy, many congressional Democrats have \"decided to " +
"keep quiet and not go there,\" said Alan Lizotte, dean and professor at the State University of New York at " +
- "Albany's School of Criminal Justice.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "politics", Field.Store.YES));
+ "Albany's School of Criminal Justice.", ft));
+ doc.add(new Field(classFieldName, "politics", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "Standing amongst the thousands of people at the state Capitol, Jorstad, director of " +
+ doc.add(new Field(textFieldName, "Standing amongst the thousands of people at the state Capitol, Jorstad, director of " +
"technology at the University of Wisconsin-La Crosse, documented the historic moment and shared it with the " +
- "world through the Internet.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "technology", Field.Store.YES));
+ "world through the Internet.", ft));
+ doc.add(new Field(classFieldName, "technology", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "So, about all those experts and analysts who've spent the past year or so saying " +
- "Facebook was going to make a phone. A new expert has stepped forward to say it's not going to happen.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "technology", Field.Store.YES));
+ doc.add(new Field(textFieldName, "So, about all those experts and analysts who've spent the past year or so saying " +
+ "Facebook was going to make a phone. A new expert has stepped forward to say it's not going to happen.", ft));
+ doc.add(new Field(classFieldName, "technology", ft));
indexWriter.addDocument(doc, analyzer);
doc = new Document();
- doc.add(new TextField(textFieldName, "More than 400 million people trust Google with their e-mail, and 50 million store files" +
+ doc.add(new Field(textFieldName, "More than 400 million people trust Google with their e-mail, and 50 million store files" +
" in the cloud using the Dropbox service. People manage their bank accounts, pay bills, trade stocks and " +
- "generally transfer or store huge volumes of personal data online.", Field.Store.YES));
- doc.add(new TextField(classFieldName, "technology", Field.Store.YES));
+ "generally transfer or store huge volumes of personal data online.", ft));
+ doc.add(new Field(classFieldName, "technology", ft));
indexWriter.addDocument(doc, analyzer);
indexWriter.commit();
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java Fri Jan 18 18:30:54 2013
@@ -212,6 +212,7 @@ public class FixedGapTermsIndexWriter ex
}
}
+ @Override
public void close() throws IOException {
boolean success = false;
try {
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/TermsIndexReaderBase.java Fri Jan 18 18:30:54 2013
@@ -42,6 +42,7 @@ public abstract class TermsIndexReaderBa
public abstract FieldIndexEnum getFieldEnum(FieldInfo fieldInfo);
+ @Override
public abstract void close() throws IOException;
public abstract boolean supportsOrd();
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java Fri Jan 18 18:30:54 2013
@@ -288,6 +288,7 @@ public class VariableGapTermsIndexWriter
}
}
+ @Override
public void close() throws IOException {
try {
final long dirStart = out.getFilePointer();
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Fri Jan 18 18:30:54 2013
@@ -133,6 +133,7 @@ public final class BloomFilteringPosting
super(BLOOM_CODEC_NAME);
}
+ @Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state)
throws IOException {
if (delegatePostingsFormat == null) {
@@ -144,6 +145,7 @@ public final class BloomFilteringPosting
delegatePostingsFormat);
}
+ @Override
public FieldsProducer fieldsProducer(SegmentReadState state)
throws IOException {
return new BloomFilteredFieldsProducer(state);
@@ -188,14 +190,17 @@ public final class BloomFilteringPosting
}
}
+ @Override
public Iterator<String> iterator() {
return delegateFieldsProducer.iterator();
}
+ @Override
public void close() throws IOException {
delegateFieldsProducer.close();
}
+ @Override
public Terms terms(String field) throws IOException {
FuzzySet filter = bloomsByFieldName.get(field);
if (filter == null) {
@@ -209,6 +214,7 @@ public final class BloomFilteringPosting
}
}
+ @Override
public int size() {
return delegateFieldsProducer.size();
}
@@ -230,26 +236,22 @@ public final class BloomFilteringPosting
@Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
- TermsEnum result;
if ((reuse != null) && (reuse instanceof BloomFilteredTermsEnum)) {
// recycle the existing BloomFilteredTermsEnum by asking the delegate
// to recycle its contained TermsEnum
BloomFilteredTermsEnum bfte = (BloomFilteredTermsEnum) reuse;
if (bfte.filter == filter) {
- bfte.delegateTermsEnum = delegateTerms
- .iterator(bfte.delegateTermsEnum);
+ bfte.reset(delegateTerms, bfte.delegateTermsEnum);
return bfte;
}
}
// We have been handed something we cannot reuse (either null, wrong
// class or wrong filter) so allocate a new object
- result = new BloomFilteredTermsEnum(delegateTerms.iterator(reuse),
- filter);
- return result;
+ return new BloomFilteredTermsEnum(delegateTerms, reuse, filter);
}
@Override
- public Comparator<BytesRef> getComparator() throws IOException {
+ public Comparator<BytesRef> getComparator() {
return delegateTerms.getComparator();
}
@@ -289,24 +291,43 @@ public final class BloomFilteringPosting
}
}
- class BloomFilteredTermsEnum extends TermsEnum {
+ final class BloomFilteredTermsEnum extends TermsEnum {
+ private Terms delegateTerms;
+ private TermsEnum delegateTermsEnum;
+ private TermsEnum reuseDelegate;
+ private final FuzzySet filter;
+
+ public BloomFilteredTermsEnum(Terms delegateTerms, TermsEnum reuseDelegate, FuzzySet filter) throws IOException {
+ this.delegateTerms = delegateTerms;
+ this.reuseDelegate = reuseDelegate;
+ this.filter = filter;
+ }
- TermsEnum delegateTermsEnum;
- private FuzzySet filter;
+ void reset(Terms delegateTerms, TermsEnum reuseDelegate) throws IOException {
+ this.delegateTerms = delegateTerms;
+ this.reuseDelegate = reuseDelegate;
+ this.delegateTermsEnum = null;
+ }
- public BloomFilteredTermsEnum(TermsEnum iterator, FuzzySet filter) {
- this.delegateTermsEnum = iterator;
- this.filter = filter;
+ private final TermsEnum delegate() throws IOException {
+ if (delegateTermsEnum == null) {
+ /* pull the iterator only if we really need it -
+ * this can be a relativly heavy operation depending on the
+ * delegate postings format and they underlying directory
+ * (clone IndexInput) */
+ delegateTermsEnum = delegateTerms.iterator(reuseDelegate);
+ }
+ return delegateTermsEnum;
}
@Override
public final BytesRef next() throws IOException {
- return delegateTermsEnum.next();
+ return delegate().next();
}
@Override
public final Comparator<BytesRef> getComparator() {
- return delegateTermsEnum.getComparator();
+ return delegateTerms.getComparator();
}
@Override
@@ -320,51 +341,51 @@ public final class BloomFilteringPosting
if (filter.contains(text) == ContainsResult.NO) {
return false;
}
- return delegateTermsEnum.seekExact(text, useCache);
+ return delegate().seekExact(text, useCache);
}
@Override
public final SeekStatus seekCeil(BytesRef text, boolean useCache)
throws IOException {
- return delegateTermsEnum.seekCeil(text, useCache);
+ return delegate().seekCeil(text, useCache);
}
@Override
public final void seekExact(long ord) throws IOException {
- delegateTermsEnum.seekExact(ord);
+ delegate().seekExact(ord);
}
@Override
public final BytesRef term() throws IOException {
- return delegateTermsEnum.term();
+ return delegate().term();
}
@Override
public final long ord() throws IOException {
- return delegateTermsEnum.ord();
+ return delegate().ord();
}
@Override
public final int docFreq() throws IOException {
- return delegateTermsEnum.docFreq();
+ return delegate().docFreq();
}
@Override
public final long totalTermFreq() throws IOException {
- return delegateTermsEnum.totalTermFreq();
+ return delegate().totalTermFreq();
}
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs,
DocsAndPositionsEnum reuse, int flags) throws IOException {
- return delegateTermsEnum.docsAndPositions(liveDocs, reuse, flags);
+ return delegate().docsAndPositions(liveDocs, reuse, flags);
}
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
throws IOException {
- return delegateTermsEnum.docs(liveDocs, reuse, flags);
+ return delegate().docs(liveDocs, reuse, flags);
}
@@ -377,12 +398,10 @@ public final class BloomFilteringPosting
private Map<FieldInfo,FuzzySet> bloomFilters = new HashMap<FieldInfo,FuzzySet>();
private SegmentWriteState state;
- // private PostingsFormat delegatePostingsFormat;
public BloomFilteredFieldsConsumer(FieldsConsumer fieldsConsumer,
SegmentWriteState state, PostingsFormat delegatePostingsFormat) {
this.delegateFieldsConsumer = fieldsConsumer;
- // this.delegatePostingsFormat=delegatePostingsFormat;
this.state = state;
}
@@ -460,10 +479,12 @@ public final class BloomFilteringPosting
this.bloomFilter = bloomFilter;
}
+ @Override
public PostingsConsumer startTerm(BytesRef text) throws IOException {
return delegateTermsConsumer.startTerm(text);
}
+ @Override
public void finishTerm(BytesRef text, TermStats stats) throws IOException {
// Record this term in our BloomFilter
@@ -473,11 +494,13 @@ public final class BloomFilteringPosting
delegateTermsConsumer.finishTerm(text, stats);
}
+ @Override
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount)
throws IOException {
delegateTermsConsumer.finish(sumTotalTermFreq, sumDocFreq, docCount);
}
+ @Override
public Comparator<BytesRef> getComparator() throws IOException {
return delegateTermsConsumer.getComparator();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Fri Jan 18 18:30:54 2013
@@ -637,6 +637,7 @@ public final class DirectPostingsFormat
termOrd = -1;
}
+ @Override
public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
@@ -1032,6 +1033,7 @@ public final class DirectPostingsFormat
}
}
+ @Override
public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Fri Jan 18 18:30:54 2013
@@ -70,10 +70,6 @@ import org.apache.lucene.util.packed.Pac
* queries that rely on advance will (AND BooleanQuery,
* PhraseQuery) will be relatively slow!
*
- * <p><b>NOTE</b>: this codec cannot address more than ~2.1 GB
- * of postings, because the underlying FST uses an int
- * to address the underlying byte[].
- *
* @lucene.experimental */
// TODO: Maybe name this 'Cached' or something to reflect
@@ -88,6 +84,13 @@ public final class MemoryPostingsFormat
this(false, PackedInts.DEFAULT);
}
+ /**
+ * Create MemoryPostingsFormat, specifying advanced FST options.
+ * @param doPackFST true if a packed FST should be built.
+ * NOTE: packed FSTs are limited to ~2.1 GB of postings.
+ * @param acceptableOverheadRatio allowable overhead for packed ints
+ * during FST construction.
+ */
public MemoryPostingsFormat(boolean doPackFST, float acceptableOverheadRatio) {
super("Memory");
this.doPackFST = doPackFST;
@@ -113,7 +116,7 @@ public final class MemoryPostingsFormat
this.field = field;
this.doPackFST = doPackFST;
this.acceptableOverheadRatio = acceptableOverheadRatio;
- builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio);
+ builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio, true, 15);
}
private class PostingsWriter extends PostingsConsumer {
@@ -269,9 +272,6 @@ public final class MemoryPostingsFormat
out.writeVLong(sumDocFreq);
out.writeVInt(docCount);
FST<BytesRef> fst = builder.finish();
- if (doPackFST) {
- fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4), acceptableOverheadRatio);
- }
fst.save(out);
//System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java Fri Jan 18 18:30:54 2013
@@ -609,6 +609,7 @@ public class PulsingPostingsReader exten
private final Map<PulsingPostingsReader,DocsEnum> enums =
new IdentityHashMap<PulsingPostingsReader,DocsEnum>();
+ @Override
public Map<PulsingPostingsReader,DocsEnum> enums() {
return enums;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java Fri Jan 18 18:30:54 2013
@@ -31,6 +31,7 @@ public abstract class IntIndexInput impl
public abstract Reader reader() throws IOException;
+ @Override
public abstract void close() throws IOException;
public abstract Index index() throws IOException;
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexOutput.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexOutput.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/IntIndexOutput.java Fri Jan 18 18:30:54 2013
@@ -56,5 +56,6 @@ public abstract class IntIndexOutput imp
* this and interact with the returned IndexWriter. */
public abstract Index index() throws IOException;
+ @Override
public abstract void close() throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Fri Jan 18 18:30:54 2013
@@ -24,13 +24,12 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
-import java.util.TreeSet;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
@@ -40,6 +39,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.StringHelper;
@@ -67,10 +67,17 @@ class SimpleTextFieldsReader extends Fie
final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD;
public SimpleTextFieldsReader(SegmentReadState state) throws IOException {
- in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context);
-
fieldInfos = state.fieldInfos;
- fields = readFields(in.clone());
+ in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context);
+ boolean success = false;
+ try {
+ fields = readFields(in.clone());
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(this);
+ }
+ }
}
private TreeMap<String,Long> readFields(IndexInput in) throws IOException {
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java Fri Jan 18 18:30:54 2013
@@ -79,6 +79,7 @@ public class SimpleTextPerDocProducer ex
return docValues;
}
+ @Override
protected DocValues loadDocValues(int docCount, Directory dir, String id,
DocValues.Type type, IOContext context) throws IOException {
return new SimpleTextDocValues(dir, context, type, id, docCount, comp, segmentSuffix);
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Fri Jan 18 18:30:54 2013
@@ -272,7 +272,7 @@ public class SimpleTextTermVectorsReader
}
@Override
- public Comparator<BytesRef> getComparator() throws IOException {
+ public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java Fri Jan 18 18:30:54 2013
@@ -87,7 +87,7 @@ public class Test10KPulsings extends Luc
for (int i = 0; i < 10050; i++) {
String expected = df.format(i);
assertEquals(expected, te.next().utf8ToString());
- de = _TestUtil.docs(random(), te, null, de, 0);
+ de = _TestUtil.docs(random(), te, null, de, DocsEnum.FLAG_NONE);
assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
}
@@ -145,7 +145,7 @@ public class Test10KPulsings extends Luc
for (int i = 0; i < 10050; i++) {
String expected = df.format(i);
assertEquals(expected, te.next().utf8ToString());
- de = _TestUtil.docs(random(), te, null, de, 0);
+ de = _TestUtil.docs(random(), te, null, de, DocsEnum.FLAG_NONE);
assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java Fri Jan 18 18:30:54 2013
@@ -60,7 +60,7 @@ public class TestPulsingReuse extends Lu
Map<DocsEnum,Boolean> allEnums = new IdentityHashMap<DocsEnum,Boolean>();
TermsEnum te = segment.terms("foo").iterator(null);
while (te.next() != null) {
- reuse = te.docs(null, reuse, 0);
+ reuse = te.docs(null, reuse, DocsEnum.FLAG_NONE);
allEnums.put(reuse, true);
}
@@ -101,7 +101,7 @@ public class TestPulsingReuse extends Lu
Map<DocsEnum,Boolean> allEnums = new IdentityHashMap<DocsEnum,Boolean>();
TermsEnum te = segment.terms("foo").iterator(null);
while (te.next() != null) {
- reuse = te.docs(null, reuse, 0);
+ reuse = te.docs(null, reuse, DocsEnum.FLAG_NONE);
allEnums.put(reuse, true);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/common-build.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/common-build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/common-build.xml Fri Jan 18 18:30:54 2013
@@ -59,8 +59,6 @@
<property name="common.classpath.excludes" value="**/*.txt,**/*.template,**/*.sha1" />
<property name="ivy.bootstrap.version" value="2.2.0" />
- <property name="ivy.resource" value="org/apache/ivy/ant/antlib.xml" />
- <available resource="${ivy.resource}" property="ivy.available" />
<property name="ivy.default.configuration" value="*"/>
<property name="ivy.sync" value="true"/>
@@ -82,7 +80,11 @@
<property name="args" value=""/>
<property name="tests.seed" value="" />
+
+ <!-- This is a hack to be able to override the JVM count for special modules that don't like parallel tests: -->
<property name="tests.jvms" value="auto" />
+ <property name="tests.jvms.override" value="${tests.jvms}" />
+
<property name="tests.multiplier" value="1" />
<property name="tests.codec" value="random" />
<property name="tests.postingsformat" value="random" />
@@ -104,14 +106,14 @@
</condition>
<property name="tests.heapsize" value="512M"/>
- <condition property="tests.clover.args" value="-XX:ReservedCodeCacheSize=128m">
+ <condition property="tests.clover.args" value="-XX:ReservedCodeCacheSize=128m -XX:MaxPermSize=192m">
<isset property="run.clover"/>
</condition>
<property name="tests.clover.args" value=""/>
<property name="tests.heapdump.args" value=""/>
- <property name="tests.tempDir" location="${build.dir}/test"/>
+ <property name="tests.workDir" location="${build.dir}/test"/>
<property name="tests.cachefile" location="${common.dir}/tools/junit4/cached-timehints.txt" />
<property name="tests.cachefilehistory" value="20" />
@@ -175,7 +177,6 @@
<property name="build.dir" location="build"/>
<!-- Needed in case a module needs the original build, also for compile-tools to be called from a module -->
<property name="common.build.dir" location="${common.dir}/build"/>
- <property name="tests.lockdir" location="${common.build.dir}"/>
<property name="dist.dir" location="${common.dir}/dist"/>
<property name="maven.dist.dir" location="${dist.dir}/maven"/>
<makeurl file="${maven.dist.dir}" property="m2.repository.url" validate="false"/>
@@ -309,7 +310,7 @@
<!-- <property name="ivy.settings.uptodate" value="true"/> -->
</target>
- <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="resolve" depends="ivy-availability-check,ivy-configure">
<!-- todo, make this a property or something.
only special cases need bundles -->
<ivy:retrieve type="jar,bundle" log="download-only"
@@ -323,6 +324,13 @@
<property name="ivy_checksum_sha1" value="f9d1e83e82fc085093510f7d2e77d81d52bc2081"/>
<target name="ivy-availability-check" unless="ivy.available">
+ <condition property="ivy.available">
+ <typefound uri="antlib:org.apache.ivy.ant" name="configure" />
+ </condition>
+ <antcall target="ivy-fail" />
+ </target>
+
+ <target name="ivy-fail" unless="ivy.available">
<echo>
This build requires Ivy and Ivy could not be found in your ant classpath.
@@ -356,10 +364,9 @@
ant config to allow the user lib to be loaded. See the wiki for more details:
http://wiki.apache.org/lucene-java/HowToContribute#antivy
</echo>
- </target>
- <target name="ivy-fail" unless="ivy.available">
<fail>Ivy is not available</fail>
</target>
+
<target name="ivy-bootstrap" description="Download and install Ivy in the users ant lib dir" depends="ivy-bootstrap1,ivy-bootstrap2,ivy-checksum"/>
<!-- try to download from repo1.maven.org -->
@@ -467,6 +474,10 @@
<artifact:install-provider artifactId="wagon-ssh" version="1.0-beta-7"/>
<parent-poms/>
<artifact:pom id="maven.project" file="@{pom.xml}"/>
+ <artifact:install file="@{jar.file}">
+ <artifact-attachments/>
+ <pom refid="maven.project"/>
+ </artifact:install>
<artifact:deploy file="@{jar.file}">
<artifact-attachments/>
<remoteRepository id="${m2.repository.id}" url="${m2.repository.url}">
@@ -749,7 +760,7 @@
<attribute name="junit.output.dir" default="${junit.output.dir}"/>
<attribute name="junit.classpath" default="junit.classpath"/>
<attribute name="testsDir" default="${build.dir}/classes/test"/>
- <attribute name="tempDir" default="${tests.tempDir}"/>
+ <attribute name="workDir" default="${tests.workDir}"/>
<attribute name="threadNum" default="1"/>
<attribute name="tests.nightly" default="${tests.nightly}"/>
<attribute name="tests.weekly" default="${tests.weekly}"/>
@@ -806,9 +817,12 @@
-->
</junit4:pickfromlist>
+ <!-- junit4 does not create this directory. TODO: is this a bug / inconsistency with dir="..."? -->
+ <mkdir dir="@{workDir}/temp" />
+
<junit4:junit4
- dir="@{tempDir}"
- tempdir="@{tempDir}"
+ dir="@{workDir}"
+ tempdir="@{workDir}/temp"
maxmemory="${tests.heapsize}"
parallelism="@{threadNum}"
@@ -846,8 +860,6 @@
<sysproperty key="tests.verbose" value="${tests.verbose}"/>
<!-- even more debugging -->
<sysproperty key="tests.infostream" value="${tests.infostream}"/>
- <!-- directory for formatter lock -->
- <sysproperty key="tests.lockdir" value="${tests.lockdir}"/>
<!-- set the codec tests should run with -->
<sysproperty key="tests.codec" value="${tests.codec}"/>
<!-- set the postingsformat tests should run with -->
@@ -884,10 +896,10 @@
<sysproperty key="java.io.tmpdir" value="." />
<!-- Restrict access to certain Java features and install security manager: -->
- <sysproperty key="tests.sandbox.dir" value="${build.dir}" />
- <sysproperty key="clover.db.dir" value="${clover.db.dir}" />
+ <sysproperty key="junit4.tempDir" file="@{workDir}/temp" />
+ <sysproperty key="clover.db.dir" file="${clover.db.dir}" />
<sysproperty key="java.security.manager" value="org.apache.lucene.util.TestSecurityManager" />
- <sysproperty key="java.security.policy" value="${common.dir}/tools/junit4/tests.policy" />
+ <sysproperty key="java.security.policy" file="${common.dir}/tools/junit4/tests.policy" />
<sysproperty key="lucene.version" value="${dev.version}"/>
@@ -904,6 +916,7 @@
<propertyref prefix="tests.failfast" />
<propertyref prefix="tests.badapples" />
<propertyref prefix="tests.timeoutSuite" />
+ <propertyref prefix="tests.jettyConnector" />
</syspropertyset>
<!-- Pass randomized settings to the forked JVM. -->
@@ -1160,7 +1173,7 @@ ${tests-output}/junit4-*.suites - pe
jars so we just order it after compile-test to ensure that -->
<target name="test" depends="clover,compile-test,install-junit4-taskdef,validate" description="Runs unit tests">
<mkdir dir="${junit.output.dir}"/>
- <test-macro threadNum="${tests.jvms}" />
+ <test-macro threadNum="${tests.jvms.override}" />
</target>
<!--
@@ -1276,7 +1289,7 @@ ${tests-output}/junit4-*.suites - pe
<fail message="You must redefine the javadocs task to do something!!!!!"/>
</target>
- <target name="install-maven-tasks" unless="maven-tasks.uptodate" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="install-maven-tasks" unless="maven-tasks.uptodate" depends="ivy-availability-check,ivy-configure">
<property name="maven-tasks.uptodate" value="true"/>
<ivy:cachepath organisation="org.apache.maven" module="maven-ant-tasks" revision="2.1.3"
inline="true" conf="master" type="jar" pathid="maven-ant-tasks.classpath"/>
@@ -1336,10 +1349,11 @@ ${tests-output}/junit4-*.suites - pe
</m2-deploy>
</sequential>
</target>
-
- <target name="-validate-maven-dependencies">
+
+ <target name="-validate-maven-dependencies.init">
+ <!-- find the correct pom.xml path and assigns it to property pom.xml -->
<property name="top.level.dir" location="${common.dir}/.."/>
- <pathconvert property="pom.xml">
+ <pathconvert property="maven.pom.xml">
<mapper>
<chainedmapper>
<globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
@@ -1348,9 +1362,23 @@ ${tests-output}/junit4-*.suites - pe
</mapper>
<path location="${ant.file}"/>
</pathconvert>
- <m2-validate-dependencies pom.xml="${pom.xml}" licenseDirectory="${license.dir}">
+
+ <!-- convert ${version} to be a glob pattern, so snapshot versions are allowed: -->
+ <loadresource property="maven.version.glob">
+ <propertyresource name="version"/>
+ <filterchain>
+ <tokenfilter>
+ <filetokenizer/>
+ <replacestring from="-SNAPSHOT" to="-*"/>
+ </tokenfilter>
+ </filterchain>
+ </loadresource>
+ </target>
+
+ <target name="-validate-maven-dependencies" depends="-validate-maven-dependencies.init">
+ <m2-validate-dependencies pom.xml="${maven.pom.xml}" licenseDirectory="${license.dir}">
<excludes>
- <rsel:name name="**/lucene-*.jar" handledirsep="true"/>
+ <rsel:name name="**/lucene-*-${maven.version.glob}.jar" handledirsep="true"/>
</excludes>
</m2-validate-dependencies>
</target>
@@ -1534,7 +1562,7 @@ ${tests-output}/junit4-*.suites - pe
</ecj-macro>
</target>
- <target name="-ecj-resolve" unless="ecj.loaded" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="-ecj-resolve" unless="ecj.loaded" depends="ivy-availability-check,ivy-configure">
<ivy:cachepath organisation="org.eclipse.jdt.core.compiler" module="ecj" revision="3.7.2"
inline="true" conf="master" type="jar" pathid="ecj.classpath" />
<componentdef classname="org.eclipse.jdt.core.JDTCompilerAdapter"
@@ -1868,7 +1896,7 @@ ${tests-output}/junit4-*.suites - pe
</macrodef>
<!-- GROOVY scripting engine for ANT tasks -->
- <target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-fail,ivy-configure">
+ <target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-configure">
<ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.0.4"
inline="true" conf="default" type="jar" transitive="true" pathid="groovy.classpath"/>
<property name="groovy.loaded" value="true"/>
@@ -1876,8 +1904,8 @@ ${tests-output}/junit4-*.suites - pe
<!-- PEGDOWN macro: Before using depend on the target "resolve-pegdown" -->
- <target name="resolve-pegdown" unless="pegdown.loaded" depends="ivy-availability-check,ivy-fail,ivy-configure">
- <ivy:cachepath organisation="org.pegdown" module="pegdown" revision="1.1.0"
+ <target name="resolve-pegdown" unless="pegdown.loaded" depends="ivy-availability-check,ivy-configure">
+ <ivy:cachepath organisation="org.pegdown" module="pegdown" revision="1.2.1"
inline="true" conf="default" type="jar" transitive="true" pathid="pegdown.classpath"/>
<property name="pegdown.loaded" value="true"/>
</target>
@@ -2031,7 +2059,7 @@ The following arguments can be provided
<junit4:pickseed property="pitest.seed" />
- <property name="pitest.sysprops" value="-Dlucene.version=${dev.version},-Dtest.seed=${pitest.seed},-Djava.security.manager,-Djava.security.policy=${common.dir}/tools/junit4/tests.policy,-Djava.io.tmpdir=${tests.tempDir},-Dtests.sandbox.dir=${build.dir}" />
+ <property name="pitest.sysprops" value="-Dlucene.version=${dev.version},-Dtest.seed=${pitest.seed},-Djava.security.manager=org.apache.lucene.util.TestSecurityManager,-Djava.security.policy=${common.dir}/tools/junit4/tests.policy,-Djava.io.tmpdir=${tests.workDir},-Djunit4.childvm.cwd=${tests.workDir},-Djunit4.tempDir=${tests.workDir}" />
<pitest
classPath="pitest.classpath"
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java Fri Jan 18 18:30:54 2013
@@ -186,6 +186,7 @@ public abstract class Analyzer implement
}
/** Frees persistent resources used by this Analyzer */
+ @Override
public void close() {
reuseStrategy.close();
}
@@ -334,6 +335,7 @@ public abstract class Analyzer implement
/**
* Closes the ReuseStrategy, freeing any resources
*/
+ @Override
public void close() {
if (storedValue != null) {
storedValue.close();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java Fri Jan 18 18:30:54 2013
@@ -153,10 +153,12 @@ public final class NumericTokenStream ex
*/
public NumericTermAttributeImpl() {}
+ @Override
public BytesRef getBytesRef() {
return bytes;
}
+ @Override
public int fillBytesRef() {
try {
assert valueSize == 64 || valueSize == 32;
@@ -170,15 +172,21 @@ public final class NumericTokenStream ex
}
}
+ @Override
public int getShift() { return shift; }
+ @Override
public void setShift(int shift) { this.shift = shift; }
+ @Override
public int incShift() {
return (shift += precisionStep);
}
+ @Override
public long getRawValue() { return value & ~((1L << shift) - 1L); }
+ @Override
public int getValueSize() { return valueSize; }
+ @Override
public void init(long value, int valueSize, int precisionStep, int shift) {
this.value = value;
this.valueSize = valueSize;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java Fri Jan 18 18:30:54 2013
@@ -242,6 +242,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see PositionIncrementAttribute
*/
+ @Override
public void setPositionIncrement(int positionIncrement) {
if (positionIncrement < 0)
throw new IllegalArgumentException
@@ -253,6 +254,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see PositionIncrementAttribute
*/
+ @Override
public int getPositionIncrement() {
return positionIncrement;
}
@@ -279,6 +281,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see OffsetAttribute
*/
+ @Override
public final int startOffset() {
return startOffset;
}
@@ -287,6 +290,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see OffsetAttribute
*/
+ @Override
public final int endOffset() {
return endOffset;
}
@@ -295,6 +299,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see OffsetAttribute
*/
+ @Override
public void setOffset(int startOffset, int endOffset) {
checkOffsets(startOffset, endOffset);
this.startOffset = startOffset;
@@ -305,6 +310,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see TypeAttribute
*/
+ @Override
public final String type() {
return type;
}
@@ -313,6 +319,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see TypeAttribute
*/
+ @Override
public final void setType(String type) {
this.type = type;
}
@@ -321,6 +328,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see FlagsAttribute
*/
+ @Override
public int getFlags() {
return flags;
}
@@ -329,6 +337,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see FlagsAttribute
*/
+ @Override
public void setFlags(int flags) {
this.flags = flags;
}
@@ -337,6 +346,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see PayloadAttribute
*/
+ @Override
public BytesRef getPayload() {
return this.payload;
}
@@ -345,6 +355,7 @@ public class Token extends CharTermAttri
* {@inheritDoc}
* @see PayloadAttribute
*/
+ @Override
public void setPayload(BytesRef payload) {
this.payload = payload;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java Fri Jan 18 18:30:54 2013
@@ -176,6 +176,7 @@ public abstract class TokenStream extend
public void reset() throws IOException {}
/** Releases resources associated with this stream. */
+ @Override
public void close() throws IOException {}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html Fri Jan 18 18:30:54 2013
@@ -230,7 +230,7 @@ and proximity searches (though sentence
create, or a combination of existing and newly created components. Before
pursuing this approach, you may find it worthwhile to explore the
<a href="{@docRoot}/../analyzers-common/overview-summary.html">analyzers-common</a> library and/or ask on the
- <a href="http://lucene.apache.org/java/docs/mailinglists.html"
+ <a href="http://lucene.apache.org/core/discussion.html"
>java-user@lucene.apache.org mailing list</a> first to see if what you
need already exists. If you are still committed to creating your own
Analyzer, have a look at the source code of any one of the many samples
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java Fri Jan 18 18:30:54 2013
@@ -68,8 +68,11 @@ public interface CharTermAttribute exten
public CharTermAttribute setEmpty();
// the following methods are redefined to get rid of IOException declaration:
+ @Override
public CharTermAttribute append(CharSequence csq);
+ @Override
public CharTermAttribute append(CharSequence csq, int start, int end);
+ @Override
public CharTermAttribute append(char c);
/** Appends the specified {@code String} to this character sequence.
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -36,16 +36,19 @@ public class CharTermAttributeImpl exten
/** Initialize this attribute with empty term text */
public CharTermAttributeImpl() {}
+ @Override
public final void copyBuffer(char[] buffer, int offset, int length) {
growTermBuffer(length);
System.arraycopy(buffer, offset, termBuffer, 0, length);
termLength = length;
}
+ @Override
public final char[] buffer() {
return termBuffer;
}
+ @Override
public final char[] resizeBuffer(int newSize) {
if(termBuffer.length < newSize){
// Not big enough; create a new array with slight
@@ -65,6 +68,7 @@ public class CharTermAttributeImpl exten
}
}
+ @Override
public final CharTermAttribute setLength(int length) {
if (length > termBuffer.length)
throw new IllegalArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.length + ")");
@@ -72,6 +76,7 @@ public class CharTermAttributeImpl exten
return this;
}
+ @Override
public final CharTermAttribute setEmpty() {
termLength = 0;
return this;
@@ -81,26 +86,31 @@ public class CharTermAttributeImpl exten
private BytesRef bytes = new BytesRef(MIN_BUFFER_SIZE);
// not until java 6 @Override
+ @Override
public int fillBytesRef() {
return UnicodeUtil.UTF16toUTF8WithHash(termBuffer, 0, termLength, bytes);
}
// not until java 6 @Override
+ @Override
public BytesRef getBytesRef() {
return bytes;
}
// *** CharSequence interface ***
+ @Override
public final int length() {
return termLength;
}
+ @Override
public final char charAt(int index) {
if (index >= termLength)
throw new IndexOutOfBoundsException();
return termBuffer[index];
}
+ @Override
public final CharSequence subSequence(final int start, final int end) {
if (start > termLength || end > termLength)
throw new IndexOutOfBoundsException();
@@ -109,12 +119,14 @@ public class CharTermAttributeImpl exten
// *** Appendable interface ***
+ @Override
public final CharTermAttribute append(CharSequence csq) {
if (csq == null) // needed for Appendable compliance
return appendNull();
return append(csq, 0, csq.length());
}
+ @Override
public final CharTermAttribute append(CharSequence csq, int start, int end) {
if (csq == null) // needed for Appendable compliance
csq = "null";
@@ -151,6 +163,7 @@ public class CharTermAttributeImpl exten
}
}
+ @Override
public final CharTermAttribute append(char c) {
resizeBuffer(termLength + 1)[termLength++] = c;
return this;
@@ -158,6 +171,7 @@ public class CharTermAttributeImpl exten
// *** For performance some convenience methods in addition to CSQ's ***
+ @Override
public final CharTermAttribute append(String s) {
if (s == null) // needed for Appendable compliance
return appendNull();
@@ -167,6 +181,7 @@ public class CharTermAttributeImpl exten
return this;
}
+ @Override
public final CharTermAttribute append(StringBuilder s) {
if (s == null) // needed for Appendable compliance
return appendNull();
@@ -176,6 +191,7 @@ public class CharTermAttributeImpl exten
return this;
}
+ @Override
public final CharTermAttribute append(CharTermAttribute ta) {
if (ta == null) // needed for Appendable compliance
return appendNull();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -26,10 +26,12 @@ public class FlagsAttributeImpl extends
/** Initialize this attribute with no bits set */
public FlagsAttributeImpl() {}
+ @Override
public int getFlags() {
return flags;
}
+ @Override
public void setFlags(int flags) {
this.flags = flags;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -53,10 +53,12 @@ public final class KeywordAttributeImpl
return keyword == other.keyword;
}
+ @Override
public boolean isKeyword() {
return keyword;
}
+ @Override
public void setKeyword(boolean isKeyword) {
keyword = isKeyword;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -27,10 +27,12 @@ public class OffsetAttributeImpl extends
/** Initialize this attribute with startOffset and endOffset of 0. */
public OffsetAttributeImpl() {}
+ @Override
public int startOffset() {
return startOffset;
}
+ @Override
public void setOffset(int startOffset, int endOffset) {
// TODO: we could assert that this is set-once, ie,
@@ -48,6 +50,7 @@ public class OffsetAttributeImpl extends
this.endOffset = endOffset;
}
+ @Override
public int endOffset() {
return endOffset;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -36,10 +36,12 @@ public class PayloadAttributeImpl extend
this.payload = payload;
}
+ @Override
public BytesRef getPayload() {
return this.payload;
}
+ @Override
public void setPayload(BytesRef payload) {
this.payload = payload;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -26,6 +26,7 @@ public class PositionIncrementAttributeI
/** Initialize this attribute with position increment of 1 */
public PositionIncrementAttributeImpl() {}
+ @Override
public void setPositionIncrement(int positionIncrement) {
if (positionIncrement < 0) {
throw new IllegalArgumentException
@@ -34,6 +35,7 @@ public class PositionIncrementAttributeI
this.positionIncrement = positionIncrement;
}
+ @Override
public int getPositionIncrement() {
return positionIncrement;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -26,6 +26,7 @@ public class PositionLengthAttributeImpl
/** Initializes this attribute with position length of 1. */
public PositionLengthAttributeImpl() {}
+ @Override
public void setPositionLength(int positionLength) {
if (positionLength < 1) {
throw new IllegalArgumentException
@@ -34,6 +35,7 @@ public class PositionLengthAttributeImpl
this.positionLength = positionLength;
}
+ @Override
public int getPositionLength() {
return positionLength;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java Fri Jan 18 18:30:54 2013
@@ -33,10 +33,12 @@ public class TypeAttributeImpl extends A
this.type = type;
}
+ @Override
public String type() {
return type;
}
+ @Override
public void setType(String type) {
this.type = type;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java Fri Jan 18 18:30:54 2013
@@ -276,13 +276,13 @@ public class BlockTreeTermsReader extend
*/
public static class Stats {
/** How many nodes in the index FST. */
- public int indexNodeCount;
+ public long indexNodeCount;
/** How many arcs in the index FST. */
- public int indexArcCount;
+ public long indexArcCount;
/** Byte size of the index. */
- public int indexNumBytes;
+ public long indexNumBytes;
/** Total number of terms in the field. */
public long totalTermCount;
@@ -833,7 +833,7 @@ public class BlockTreeTermsReader extend
if (index == null) {
fstReader = null;
} else {
- fstReader = index.getBytesReader(0);
+ fstReader = index.getBytesReader();
}
// TODO: if the automaton is "smallish" we really
@@ -1277,7 +1277,7 @@ public class BlockTreeTermsReader extend
if (index == null) {
fstReader = null;
} else {
- fstReader = index.getBytesReader(0);
+ fstReader = index.getBytesReader();
}
// Init w/ root block; don't use index since it may
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java Fri Jan 18 18:30:54 2013
@@ -23,7 +23,6 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
@@ -41,6 +40,7 @@ import org.apache.lucene.util.fst.BytesR
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.NoOutputs;
import org.apache.lucene.util.fst.Util;
+import org.apache.lucene.util.packed.PackedInts;
/*
TODO:
@@ -187,7 +187,7 @@ public class BlockTreeTermsWriter extend
public final static int DEFAULT_MAX_BLOCK_SIZE = 48;
//public final static boolean DEBUG = false;
- private final static boolean SAVE_DOT_FILES = false;
+ //private final static boolean SAVE_DOT_FILES = false;
static final int OUTPUT_FLAGS_NUM_BITS = 2;
static final int OUTPUT_FLAGS_MASK = 0x3;
@@ -419,7 +419,8 @@ public class BlockTreeTermsWriter extend
final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
final Builder<BytesRef> indexBuilder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1,
0, 0, true, false, Integer.MAX_VALUE,
- outputs, null, false);
+ outputs, null, false,
+ PackedInts.COMPACT, true, 15);
//if (DEBUG) {
// System.out.println(" compile index for prefix=" + prefix);
//}
@@ -962,7 +963,9 @@ public class BlockTreeTermsWriter extend
0, 0, true,
true, Integer.MAX_VALUE,
noOutputs,
- new FindBlocks(), false);
+ new FindBlocks(), false,
+ PackedInts.COMPACT,
+ true, 15);
postingsWriter.setField(fieldInfo);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java Fri Jan 18 18:30:54 2013
@@ -118,6 +118,7 @@ public abstract class DocValuesArraySour
return values;
}
+ @Override
public double getFloat(int docID) {
return getInt(docID);
}
@@ -140,6 +141,7 @@ public abstract class DocValuesArraySour
return new ByteValues((byte[]) array);
}
+ @Override
public void toBytes(long value, BytesRef bytesRef) {
if (bytesRef.bytes.length == 0) {
bytesRef.bytes = new byte[1];
@@ -183,6 +185,7 @@ public abstract class DocValuesArraySour
return values;
}
+ @Override
public double getFloat(int docID) {
return getInt(docID);
}
@@ -199,6 +202,7 @@ public abstract class DocValuesArraySour
return new ShortValues(input, numDocs);
}
+ @Override
public void toBytes(long value, BytesRef bytesRef) {
copyShort(bytesRef, (short) (0xFFFFL & value));
}
@@ -243,6 +247,7 @@ public abstract class DocValuesArraySour
return values;
}
+ @Override
public double getFloat(int docID) {
return getInt(docID);
}
@@ -259,6 +264,7 @@ public abstract class DocValuesArraySour
return new IntValues(input, numDocs);
}
+ @Override
public void toBytes(long value, BytesRef bytesRef) {
copyInt(bytesRef, (int) (0xFFFFFFFF & value));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java Fri Jan 18 18:30:54 2013
@@ -54,6 +54,7 @@ public abstract class FieldsConsumer imp
public abstract TermsConsumer addField(FieldInfo field) throws IOException;
/** Called when we are done adding everything. */
+ @Override
public abstract void close() throws IOException;
/** Called during merging to merge all {@link Fields} from
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java Fri Jan 18 18:30:54 2013
@@ -66,6 +66,7 @@ public abstract class PostingsReaderBase
public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsAndPositionsEnum reuse,
int flags) throws IOException;
+ @Override
public abstract void close() throws IOException;
/** Reads data for all terms in the next block; this
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java Fri Jan 18 18:30:54 2013
@@ -38,5 +38,6 @@ public abstract class StoredFieldsReader
/** Visit the stored fields for document <code>n</code> */
public abstract void visitDocument(int n, StoredFieldVisitor visitor) throws IOException;
+ @Override
public abstract StoredFieldsReader clone();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Fri Jan 18 18:30:54 2013
@@ -55,7 +55,10 @@ public abstract class StoredFieldsWriter
* called even if the document has no stored fields, in
* this case <code>numStoredFields</code> will be zero. */
public abstract void startDocument(int numStoredFields) throws IOException;
-
+
+ /** Called when a document and all its fields have been added. */
+ public void finishDocument() throws IOException {}
+
/** Writes a single stored field. */
public abstract void writeField(FieldInfo info, StorableField field) throws IOException;
@@ -116,7 +119,10 @@ public abstract class StoredFieldsWriter
for (StorableField field : doc) {
writeField(fieldInfos.fieldInfo(field.name()), field);
}
+
+ finishDocument();
}
+ @Override
public abstract void close() throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java Fri Jan 18 18:30:54 2013
@@ -44,5 +44,6 @@ public abstract class TermVectorsReader
/** Create a clone that one caller at a time may use to
* read term vectors. */
+ @Override
public abstract TermVectorsReader clone();
}