You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/14 15:51:59 UTC
svn commit: r1103112 [4/24] - in /lucene/dev/branches/flexscoring: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ant/
dev-tools/idea/lucene/contrib/db/bdb-je/
dev-tools/idea/lucene/contrib/db/bdb/ dev-tools/idea/lucene/contr...
Modified: lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Sat May 14 13:51:35 2011
@@ -191,7 +191,7 @@ public class TestQPHelper extends Lucene
public StandardQueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
+ a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(a);
@@ -281,7 +281,7 @@ public class TestQPHelper extends Lucene
public Query getQueryDOA(String query, Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
+ a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(a);
qp.setDefaultOperator(Operator.AND);
@@ -301,7 +301,7 @@ public class TestQPHelper extends Lucene
}
public void testConstantScoreAutoRewrite() throws Exception {
- StandardQueryParser qp = new StandardQueryParser(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ StandardQueryParser qp = new StandardQueryParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
Query q = qp.parse("foo*bar", "field");
assertTrue(q instanceof WildcardQuery);
assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((MultiTermQuery) q).getRewriteMethod());
@@ -410,9 +410,9 @@ public class TestQPHelper extends Lucene
public void testSimple() throws Exception {
assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
assertQueryEquals("term term term", null, "term term term");
- assertQueryEquals("t�rm term term", new MockAnalyzer(MockTokenizer.WHITESPACE, false),
+ assertQueryEquals("t�rm term term", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false),
"t�rm term term");
- assertQueryEquals("�mlaut", new MockAnalyzer(MockTokenizer.WHITESPACE, false), "�mlaut");
+ assertQueryEquals("�mlaut", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), "�mlaut");
// FIXME: change MockAnalyzer to not extend CharTokenizer for this test
//assertQueryEquals("\"\"", new KeywordAnalyzer(), "");
@@ -470,7 +470,7 @@ public class TestQPHelper extends Lucene
}
public void testPunct() throws Exception {
- Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("a&b", a, "a&b");
assertQueryEquals("a&&b", a, "a&&b");
assertQueryEquals(".NET", a, ".NET");
@@ -491,7 +491,7 @@ public class TestQPHelper extends Lucene
assertQueryEquals("term 1.0 1 2", null, "term");
assertQueryEquals("term term1 term2", null, "term term term");
- Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("3", a, "3");
assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2");
assertQueryEquals("term term1 term2", a, "term term1 term2");
@@ -726,7 +726,7 @@ public class TestQPHelper extends Lucene
}
public void testEscaped() throws Exception {
- Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
/*
* assertQueryEquals("\\[brackets", a, "\\[brackets");
@@ -825,7 +825,7 @@ public class TestQPHelper extends Lucene
}
public void testQueryStringEscaping() throws Exception {
- Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
@@ -866,7 +866,7 @@ public class TestQPHelper extends Lucene
@Ignore("contrib queryparser shouldn't escape wildcard terms")
public void testEscapedWildcard() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
WildcardQuery q = new WildcardQuery(new Term("field", "foo\\?ba?r"));
assertEquals(q, qp.parse("foo\\?ba?r", "field"));
@@ -904,7 +904,7 @@ public class TestQPHelper extends Lucene
public void testBoost() throws Exception {
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.makeString("on"));
- Analyzer oneStopAnalyzer = new MockAnalyzer(MockTokenizer.SIMPLE, true, stopSet, true);
+ Analyzer oneStopAnalyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopSet, true);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(oneStopAnalyzer);
@@ -920,7 +920,7 @@ public class TestQPHelper extends Lucene
assertNotNull(q);
StandardQueryParser qp2 = new StandardQueryParser();
- qp2.setAnalyzer(new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ qp2.setAnalyzer(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
q = qp2.parse("the^3", "field");
// "the" is a stop word so the result is an empty query:
@@ -950,7 +950,7 @@ public class TestQPHelper extends Lucene
public void testCustomQueryParserWildcard() {
try {
- new QPTestParser(new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("a?t", "contents");
+ new QPTestParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("a?t", "contents");
fail("Wildcard queries should not be allowed");
} catch (QueryNodeException expected) {
// expected exception
@@ -959,7 +959,7 @@ public class TestQPHelper extends Lucene
public void testCustomQueryParserFuzzy() throws Exception {
try {
- new QPTestParser(new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("xunit~", "contents");
+ new QPTestParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("xunit~", "contents");
fail("Fuzzy queries should not be allowed");
} catch (QueryNodeException expected) {
// expected exception
@@ -970,7 +970,7 @@ public class TestQPHelper extends Lucene
BooleanQuery.setMaxClauseCount(2);
try {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
qp.parse("one two three", "field");
fail("ParseException expected due to too many boolean clauses");
@@ -984,7 +984,7 @@ public class TestQPHelper extends Lucene
*/
public void testPrecedence() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
Query query1 = qp.parse("A AND B OR C AND D", "field");
Query query2 = qp.parse("+A +B +C +D", "field");
@@ -995,7 +995,7 @@ public class TestQPHelper extends Lucene
// Todo: Convert from DateField to DateUtil
// public void testLocalDateFormat() throws IOException, QueryNodeException {
// Directory ramDir = newDirectory();
-// IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
+// IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
// addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw);
// addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw);
// iw.close();
@@ -1116,7 +1116,7 @@ public class TestQPHelper extends Lucene
public void testStopwords() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
- qp.setAnalyzer(new MockAnalyzer(MockTokenizer.SIMPLE, true, stopSet, true));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopSet, true));
Query result = qp.parse("a:the OR a:foo", "a");
assertNotNull("result is null and it shouldn't be", result);
@@ -1140,7 +1140,7 @@ public class TestQPHelper extends Lucene
public void testPositionIncrement() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(
- new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
+ new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
qp.setEnablePositionIncrements(true);
@@ -1161,7 +1161,7 @@ public class TestQPHelper extends Lucene
public void testMatchAllDocs() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
assertEquals(new MatchAllDocsQuery(), qp.parse("*:*", "field"));
assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)", "field"));
@@ -1173,7 +1173,7 @@ public class TestQPHelper extends Lucene
private void assertHits(int expected, String query, IndexSearcher is)
throws IOException, QueryNodeException {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
qp.setLocale(Locale.ENGLISH);
Query q = qp.parse(query, "date");
Modified: lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/surround/query/SingleFieldTestDb.java Sat May 14 13:51:35 2011
@@ -41,7 +41,7 @@ public class SingleFieldTestDb {
fieldName = fName;
IndexWriter writer = new IndexWriter(db, new IndexWriterConfig(
Version.LUCENE_CURRENT,
- new MockAnalyzer()));
+ new MockAnalyzer(random)));
for (int j = 0; j < docs.length; j++) {
Document d = new Document();
d.add(new Field(fieldName, docs[j], Field.Store.NO, Field.Index.ANALYZED));
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java Sat May 14 13:51:35 2011
@@ -71,7 +71,7 @@ public class TestCartesian extends Lucen
super.setUp();
directory = newDirectory();
- IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
setUpPlotter( 2, 15);
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java Sat May 14 13:51:35 2011
@@ -47,7 +47,7 @@ public class TestDistance extends Lucene
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
addData(writer);
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java Sat May 14 13:51:35 2011
@@ -29,7 +29,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Terms;
@@ -45,7 +45,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;
-import org.apache.lucene.util.VirtualMethod;
/**
* <p>
@@ -508,7 +507,7 @@ public class SpellChecker implements jav
ensureOpen();
final Directory dir = this.spellIndex;
final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(mergeFactor);
+ ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
IndexSearcher indexSearcher = obtainSearcher();
final List<TermsEnum> termsEnums = new ArrayList<TermsEnum>();
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestDirectSpellChecker.java Sat May 14 13:51:35 2011
@@ -35,7 +35,7 @@ public class TestDirectSpellChecker exte
spellChecker.setMinQueryLength(0);
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
- new MockAnalyzer(MockTokenizer.SIMPLE, true));
+ new MockAnalyzer(random, MockTokenizer.SIMPLE, true));
for (int i = 0; i < 20; i++) {
Document doc = new Document();
@@ -93,7 +93,7 @@ public class TestDirectSpellChecker exte
public void testOptions() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
- new MockAnalyzer(MockTokenizer.SIMPLE, true));
+ new MockAnalyzer(random, MockTokenizer.SIMPLE, true));
Document doc = new Document();
doc.add(newField("text", "foobar", Field.Store.NO, Field.Index.ANALYZED));
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Sat May 14 13:51:35 2011
@@ -46,7 +46,7 @@ public class TestLuceneDictionary extend
public void setUp() throws Exception {
super.setUp();
store = newDirectory();
- IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
+ IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
Document doc;
Modified: lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Sat May 14 13:51:35 2011
@@ -54,7 +54,7 @@ public class TestSpellChecker extends Lu
//create a user index
userindex = newDirectory();
IndexWriter writer = new IndexWriter(userindex, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()));
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)));
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Sat May 14 13:51:35 2011
@@ -116,14 +116,16 @@ public final class SynExpand {
if ( a == null) a = new StandardAnalyzer(Version.LUCENE_CURRENT);
// [1] Parse query into separate words so that when we expand we can avoid dups
- TokenStream ts = a.tokenStream( field, new StringReader( query));
+ TokenStream ts = a.reusableTokenStream( field, new StringReader( query));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
+ ts.reset();
while (ts.incrementToken()) {
String word = termAtt.toString();
if ( already.add( word))
top.add( word);
}
+ ts.end();
+ ts.close();
final BooleanQuery tmp = new BooleanQuery();
// [2] form query
Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java Sat May 14 13:51:35 2011
@@ -124,7 +124,7 @@ public class SynLookup {
List<String> top = new LinkedList<String>(); // needs to be separately listed..
// [1] Parse query into separate words so that when we expand we can avoid dups
- TokenStream ts = a.tokenStream( field, new StringReader( query));
+ TokenStream ts = a.reusableTokenStream( field, new StringReader( query));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java Sat May 14 13:51:35 2011
@@ -36,7 +36,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -250,7 +250,7 @@ public class Syns2Index
// override the specific index if it already exists
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
Version.LUCENE_CURRENT, ana).setOpenMode(OpenMode.CREATE));
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
+ ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
Iterator<String> i1 = word2Nums.keySet().iterator();
while (i1.hasNext()) // for each word
{
Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java Sat May 14 13:51:35 2011
@@ -111,7 +111,6 @@ public class TestSynonymTokenFilter exte
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
- streams.result.reset(); // reset the SynonymTokenFilter
}
return streams.result;
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java Sat May 14 13:51:35 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestWordnet extends LuceneTestCase {
private IndexSearcher searcher;
@@ -42,6 +43,7 @@ public class TestWordnet extends LuceneT
// create a temporary synonym index
File testFile = getDataFile("testSynonyms.txt");
String commandLineArgs[] = { testFile.getAbsolutePath(), storePathName };
+ _TestUtil.rmDir(new File(storePathName));
try {
Syns2Index.main(commandLineArgs);
@@ -61,7 +63,7 @@ public class TestWordnet extends LuceneT
private void assertExpandsTo(String term, String expected[]) throws IOException {
Query expandedQuery = SynExpand.expand(term, searcher, new
- MockAnalyzer(), "field", 1F);
+ MockAnalyzer(random), "field", 1F);
BooleanQuery expectedQuery = new BooleanQuery();
for (String t : expected)
expectedQuery.add(new TermQuery(new Term("field", t)),
@@ -71,8 +73,12 @@ public class TestWordnet extends LuceneT
@Override
public void tearDown() throws Exception {
- searcher.close();
- dir.close();
+ if (searcher != null) {
+ searcher.close();
+ }
+ if (dir != null) {
+ dir.close();
+ }
rmDir(storePathName); // delete our temporary synonym index
super.tearDown();
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java Sat May 14 13:51:35 2011
@@ -76,13 +76,16 @@ public class LikeThisQueryBuilder implem
stopWordsSet=new HashSet<String>();
for (int i = 0; i < fields.length; i++)
{
- TokenStream ts = analyzer.tokenStream(fields[i],new StringReader(stopWords));
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
try
{
+ TokenStream ts = analyzer.reusableTokenStream(fields[i],new StringReader(stopWords));
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
while(ts.incrementToken()) {
stopWordsSet.add(termAtt.toString());
}
+ ts.end();
+ ts.close();
}
catch(IOException ioe)
{
Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Sat May 14 13:51:35 2011
@@ -56,14 +56,17 @@ public class SpanOrTermsBuilder extends
try
{
ArrayList<SpanQuery> clausesList=new ArrayList<SpanQuery>();
- TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
+ TokenStream ts=analyzer.reusableTokenStream(fieldName,new StringReader(value));
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
+ ts.reset();
while (ts.incrementToken()) {
termAtt.fillBytesRef();
SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, new BytesRef(bytes)));
clausesList.add(stq);
}
+ ts.end();
+ ts.close();
SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f));
return soq;
Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Sat May 14 13:51:35 2011
@@ -57,13 +57,14 @@ public class TermsFilterBuilder implemen
TermsFilter tf = new TermsFilter();
String text = DOMUtils.getNonBlankTextOrFail(e);
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
- TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
try
{
+ TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text));
+ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
+ ts.reset();
while (ts.incrementToken()) {
termAtt.fillBytesRef();
if (term == null)
@@ -76,6 +77,8 @@ public class TermsFilterBuilder implemen
}
tf.addTerm(term);
}
+ ts.end();
+ ts.close();
}
catch (IOException ioe)
{
Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Sat May 14 13:51:35 2011
@@ -55,12 +55,13 @@ public class TermsQueryBuilder implement
BooleanQuery bq=new BooleanQuery(DOMUtils.getAttribute(e,"disableCoord",false));
bq.setMinimumNumberShouldMatch(DOMUtils.getAttribute(e,"minimumNumberShouldMatch",0));
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
try
{
+ TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text));
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
+ ts.reset();
while (ts.incrementToken()) {
termAtt.fillBytesRef();
if (term == null)
@@ -73,6 +74,8 @@ public class TermsQueryBuilder implement
}
bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
}
+ ts.end();
+ ts.close();
}
catch (IOException ioe)
{
Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java Sat May 14 13:51:35 2011
@@ -49,7 +49,7 @@ public class TestParser extends LuceneTe
@BeforeClass
public static void beforeClass() throws Exception {
// TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
- Analyzer analyzer=new MockAnalyzer(MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET, false);
+ Analyzer analyzer=new MockAnalyzer(random, MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET, false);
//initialize the parser
builder=new CorePlusExtensionsParser("contents",analyzer);
@@ -187,7 +187,8 @@ public class TestParser extends LuceneTe
}
public void testDuplicateFilterQueryXML() throws ParserException, IOException
{
- Assume.assumeTrue(searcher.getIndexReader().getSequentialSubReaders().length == 1);
+ Assume.assumeTrue(searcher.getIndexReader().getSequentialSubReaders() == null ||
+ searcher.getIndexReader().getSequentialSubReaders().length == 1);
Query q=parse("DuplicateFilterQuery.xml");
int h = searcher.search(q, null, 1000).totalHits;
assertEquals("DuplicateFilterQuery should produce 1 result ", 1,h);
Modified: lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java Sat May 14 13:51:35 2011
@@ -44,7 +44,7 @@ import org.xml.sax.SAXException;
public class TestQueryTemplateManager extends LuceneTestCase {
CoreParser builder;
- Analyzer analyzer=new MockAnalyzer();
+ Analyzer analyzer=new MockAnalyzer(random);
private IndexSearcher searcher;
private Directory dir;
Modified: lucene/dev/branches/flexscoring/lucene/docs/contributions.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/contributions.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/contributions.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/contributions.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Contributions
@@ -275,7 +275,7 @@ document.write("Last Published: " + docu
<a href="#PDFTextStream -- PDF text and metadata extraction">PDFTextStream -- PDF text and metadata extraction</a>
</li>
<li>
-<a href="#PJ Classic & PJ Professional - PDF Document Conversion">PJ Classic & PJ Professional - PDF Document Conversion</a>
+<a href="#PJ Classic & PJ Professional - PDF Document Conversion">PJ Classic & PJ Professional - PDF Document Conversion</a>
</li>
</ul>
</li>
@@ -403,7 +403,7 @@ document.write("Last Published: " + docu
URL
</th>
<td>
- <a href="http://marc.theaimsgroup.com/?l=lucene-dev&m=100723333506246&w=2">
+ <a href="http://marc.theaimsgroup.com/?l=lucene-dev&m=100723333506246&w=2">
http://marc.theaimsgroup.com/?l=lucene-dev&m=100723333506246&w=2
</a>
</td>
@@ -538,7 +538,7 @@ document.write("Last Published: " + docu
</tr>
</table>
-<a name="N10124"></a><a name="PJ Classic & PJ Professional - PDF Document Conversion"></a>
+<a name="N10124"></a><a name="PJ Classic & PJ Professional - PDF Document Conversion"></a>
<h3 class="boxed">PJ Classic & PJ Professional - PDF Document Conversion</h3>
<table class="ForrestTable" cellspacing="1" cellpadding="4">
Modified: lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/contributions.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/demo.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/demo.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/demo.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Building and Installing the Basic Demo
Modified: lucene/dev/branches/flexscoring/lucene/docs/demo.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/demo.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/demo.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/demo2.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo2.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/demo2.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/demo2.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Basic Demo Sources Walk-through
Modified: lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/demo2.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/fileformats.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/fileformats.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/fileformats.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Index File Formats
@@ -425,11 +425,19 @@ document.write("Last Published: " + docu
<p>
In version 3.1, segments records the code version
that created them. See LUCENE-2720 for details.
+
+ Additionally segments track explicitly whether or
+ not they have term vectors. See LUCENE-2811 for details.
+ </p>
+<p>
+ In version 3.2, numeric fields are written as natively
+ to stored fields file, previously they were stored in
+ text format only.
</p>
</div>
-<a name="N10037"></a><a name="Definitions"></a>
+<a name="N1003A"></a><a name="Definitions"></a>
<h2 class="boxed">Definitions</h2>
<div class="section">
<p>
@@ -470,7 +478,7 @@ document.write("Last Published: " + docu
strings, the first naming the field, and the second naming text
within the field.
</p>
-<a name="N10057"></a><a name="Inverted Indexing"></a>
+<a name="N1005A"></a><a name="Inverted Indexing"></a>
<h3 class="boxed">Inverted Indexing</h3>
<p>
The index stores statistics about terms in order
@@ -480,7 +488,7 @@ document.write("Last Published: " + docu
it. This is the inverse of the natural relationship, in which
documents list terms.
</p>
-<a name="N10063"></a><a name="Types of Fields"></a>
+<a name="N10066"></a><a name="Types of Fields"></a>
<h3 class="boxed">Types of Fields</h3>
<p>
In Lucene, fields may be <i>stored</i>, in which
@@ -494,7 +502,7 @@ document.write("Last Published: " + docu
to be indexed literally.
</p>
<p>See the <a href="api/core/org/apache/lucene/document/Field.html">Field</a> java docs for more information on Fields.</p>
-<a name="N10080"></a><a name="Segments"></a>
+<a name="N10083"></a><a name="Segments"></a>
<h3 class="boxed">Segments</h3>
<p>
Lucene indexes may be composed of multiple sub-indexes, or
@@ -520,7 +528,7 @@ document.write("Last Published: " + docu
Searches may involve multiple segments and/or multiple indexes, each
index potentially composed of a set of segments.
</p>
-<a name="N1009E"></a><a name="Document Numbers"></a>
+<a name="N100A1"></a><a name="Document Numbers"></a>
<h3 class="boxed">Document Numbers</h3>
<p>
Internally, Lucene refers to documents by an integer <i>document
@@ -575,7 +583,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N100C5"></a><a name="Overview"></a>
+<a name="N100C8"></a><a name="Overview"></a>
<h2 class="boxed">Overview</h2>
<div class="section">
<p>
@@ -674,7 +682,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N10108"></a><a name="File Naming"></a>
+<a name="N1010B"></a><a name="File Naming"></a>
<h2 class="boxed">File Naming</h2>
<div class="section">
<p>
@@ -701,7 +709,7 @@ document.write("Last Published: " + docu
</p>
</div>
-<a name="N10117"></a><a name="file-names"></a>
+<a name="N1011A"></a><a name="file-names"></a>
<h2 class="boxed">Summary of File Extensions</h2>
<div class="section">
<p>The following table summarizes the names and extensions of the files in Lucene:
@@ -843,10 +851,10 @@ document.write("Last Published: " + docu
</div>
-<a name="N10201"></a><a name="Primitive Types"></a>
+<a name="N10204"></a><a name="Primitive Types"></a>
<h2 class="boxed">Primitive Types</h2>
<div class="section">
-<a name="N10206"></a><a name="Byte"></a>
+<a name="N10209"></a><a name="Byte"></a>
<h3 class="boxed">Byte</h3>
<p>
The most primitive type
@@ -854,7 +862,7 @@ document.write("Last Published: " + docu
other data types are defined as sequences
of bytes, so file formats are byte-order independent.
</p>
-<a name="N1020F"></a><a name="UInt32"></a>
+<a name="N10212"></a><a name="UInt32"></a>
<h3 class="boxed">UInt32</h3>
<p>
32-bit unsigned integers are written as four
@@ -864,7 +872,7 @@ document.write("Last Published: " + docu
UInt32 --> <Byte><sup>4</sup>
</p>
-<a name="N1021E"></a><a name="Uint64"></a>
+<a name="N10221"></a><a name="Uint64"></a>
<h3 class="boxed">Uint64</h3>
<p>
64-bit unsigned integers are written as eight
@@ -873,7 +881,7 @@ document.write("Last Published: " + docu
<p>UInt64 --> <Byte><sup>8</sup>
</p>
-<a name="N1022D"></a><a name="VInt"></a>
+<a name="N10230"></a><a name="VInt"></a>
<h3 class="boxed">VInt</h3>
<p>
A variable-length format for positive integers is
@@ -1423,13 +1431,13 @@ document.write("Last Published: " + docu
This provides compression while still being
efficient to decode.
</p>
-<a name="N10512"></a><a name="Chars"></a>
+<a name="N10515"></a><a name="Chars"></a>
<h3 class="boxed">Chars</h3>
<p>
Lucene writes unicode
character sequences as UTF-8 encoded bytes.
</p>
-<a name="N1051B"></a><a name="String"></a>
+<a name="N1051E"></a><a name="String"></a>
<h3 class="boxed">String</h3>
<p>
Lucene writes strings as UTF-8 encoded bytes.
@@ -1442,10 +1450,10 @@ document.write("Last Published: " + docu
</div>
-<a name="N10528"></a><a name="Compound Types"></a>
+<a name="N1052B"></a><a name="Compound Types"></a>
<h2 class="boxed">Compound Types</h2>
<div class="section">
-<a name="N1052D"></a><a name="MapStringString"></a>
+<a name="N10530"></a><a name="MapStringString"></a>
<h3 class="boxed">Map<String,String></h3>
<p>
In a couple places Lucene stores a Map
@@ -1458,13 +1466,13 @@ document.write("Last Published: " + docu
</div>
-<a name="N1053D"></a><a name="Per-Index Files"></a>
+<a name="N10540"></a><a name="Per-Index Files"></a>
<h2 class="boxed">Per-Index Files</h2>
<div class="section">
<p>
The files in this section exist one-per-index.
</p>
-<a name="N10545"></a><a name="Segments File"></a>
+<a name="N10548"></a><a name="Segments File"></a>
<h3 class="boxed">Segments File</h3>
<p>
The active segments in the index are stored in the
@@ -1508,7 +1516,7 @@ document.write("Last Published: " + docu
<b>3.1</b>
Segments --> Format, Version, NameCounter, SegCount, <SegVersion, SegName, SegSize, DelGen, DocStoreOffset, [DocStoreSegment, DocStoreIsCompoundFile], HasSingleNormFile, NumField,
NormGen<sup>NumField</sup>,
- IsCompoundFile, DeletionCount, HasProx, Diagnostics><sup>SegCount</sup>, CommitUserData, Checksum
+ IsCompoundFile, DeletionCount, HasProx, Diagnostics, HasVectors><sup>SegCount</sup>, CommitUserData, Checksum
</p>
<p>
Format, NameCounter, SegCount, SegSize, NumField,
@@ -1525,7 +1533,7 @@ document.write("Last Published: " + docu
</p>
<p>
IsCompoundFile, HasSingleNormFile,
- DocStoreIsCompoundFile, HasProx --> Int8
+ DocStoreIsCompoundFile, HasProx, HasVectors --> Int8
</p>
<p>
CommitUserData --> Map<String,String>
@@ -1634,7 +1642,10 @@ document.write("Last Published: " + docu
Lucene version, OS, Java version, why the segment
was created (merge, flush, addIndexes), etc.
</p>
-<a name="N105CD"></a><a name="Lock File"></a>
+<p> HasVectors is 1 if this segment stores term vectors,
+ else it's 0.
+ </p>
+<a name="N105D3"></a><a name="Lock File"></a>
<h3 class="boxed">Lock File</h3>
<p>
The write lock, which is stored in the index
@@ -1648,14 +1659,14 @@ document.write("Last Published: " + docu
documents). This lock file ensures that only one
writer is modifying the index at a time.
</p>
-<a name="N105D6"></a><a name="Deletable File"></a>
+<a name="N105DC"></a><a name="Deletable File"></a>
<h3 class="boxed">Deletable File</h3>
<p>
A writer dynamically computes
the files that are deletable, instead, so no file
is written.
</p>
-<a name="N105DF"></a><a name="Compound Files"></a>
+<a name="N105E5"></a><a name="Compound Files"></a>
<h3 class="boxed">Compound Files</h3>
<p>Starting with Lucene 1.4 the compound file format became default. This
is simply a container for all files described in the next section
@@ -1682,14 +1693,14 @@ document.write("Last Published: " + docu
</div>
-<a name="N10607"></a><a name="Per-Segment Files"></a>
+<a name="N1060D"></a><a name="Per-Segment Files"></a>
<h2 class="boxed">Per-Segment Files</h2>
<div class="section">
<p>
The remaining files are all per-segment, and are
thus defined by suffix.
</p>
-<a name="N1060F"></a><a name="Fields"></a>
+<a name="N10615"></a><a name="Fields"></a>
<h3 class="boxed">Fields</h3>
<p>
@@ -1862,13 +1873,29 @@ document.write("Last Published: " + docu
<li>third bit is one for fields with compression option enabled
(if compression is enabled, the algorithm used is ZLIB),
only available for indexes until Lucene version 2.9.x</li>
+
+<li>4th to 6th bits (mask: 0x7<<3) define the type of a
+ numeric field: <ul>
+
+<li>all bits in mask are cleared if no numeric field at all</li>
+
+<li>1<<3: Value is Int</li>
+
+<li>2<<3: Value is Long</li>
+
+<li>3<<3: Value is Int as Float (as of Integer.intBitsToFloat)</li>
+
+<li>4<<3: Value is Long as Double (as of Double.longBitsToDouble)</li>
+
+</ul>
+</li>
</ul>
</p>
<p>Value -->
- String | BinaryValue (depending on Bits)
+ String | BinaryValue | Int | Long (depending on Bits)
</p>
<p>BinaryValue -->
@@ -1883,7 +1910,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N106B6"></a><a name="Term Dictionary"></a>
+<a name="N106D0"></a><a name="Term Dictionary"></a>
<h3 class="boxed">Term Dictionary</h3>
<p>
The term dictionary is represented as two files:
@@ -2075,7 +2102,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N1073A"></a><a name="Frequencies"></a>
+<a name="N10754"></a><a name="Frequencies"></a>
<h3 class="boxed">Frequencies</h3>
<p>
The .frq file contains the lists of documents
@@ -2203,7 +2230,7 @@ document.write("Last Published: " + docu
entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
to entry 31 on level 0.
</p>
-<a name="N107C2"></a><a name="Positions"></a>
+<a name="N107DC"></a><a name="Positions"></a>
<h3 class="boxed">Positions</h3>
<p>
The .prx file contains the lists of positions that
@@ -2273,7 +2300,7 @@ document.write("Last Published: " + docu
Payload. If PayloadLength is not stored, then this Payload has the same
length as the Payload at the previous position.
</p>
-<a name="N107FE"></a><a name="Normalization Factors"></a>
+<a name="N10818"></a><a name="Normalization Factors"></a>
<h3 class="boxed">Normalization Factors</h3>
<p>There's a single .nrm file containing all norms:
</p>
@@ -2353,7 +2380,7 @@ document.write("Last Published: " + docu
</p>
<p>Separate norm files are created (when adequate) for both compound and non compound segments.
</p>
-<a name="N1084F"></a><a name="Term Vectors"></a>
+<a name="N10869"></a><a name="Term Vectors"></a>
<h3 class="boxed">Term Vectors</h3>
<p>
Term Vector support is an optional on a field by
@@ -2489,7 +2516,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N108EB"></a><a name="Deleted Documents"></a>
+<a name="N10905"></a><a name="Deleted Documents"></a>
<h3 class="boxed">Deleted Documents</h3>
<p>The .del file is
optional, and only exists when a segment contains deletions.
@@ -2553,7 +2580,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N10925"></a><a name="Limitations"></a>
+<a name="N1093F"></a><a name="Limitations"></a>
<h2 class="boxed">Limitations</h2>
<div class="section">
<p>
Modified: lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/fileformats.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Getting Started Guide
@@ -268,15 +268,13 @@ may wish to skip sections.
<li>
<a href="demo.html">About the command-line Lucene demo and its usage</a>. This section
- is intended for anyone who wants to use the command-line Lucene demo.</li>
-<p></p>
+ is intended for anyone who wants to use the command-line Lucene demo.</li>
<li>
<a href="demo2.html">About the sources and implementation for the command-line Lucene
demo</a>. This section walks through the implementation details (sources) of the
- command-line Lucene demo. This section is intended for developers.</li>
-<p></p>
+ command-line Lucene demo. This section is intended for developers.</li>
</ul>
</div>
Modified: lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/gettingstarted.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/index.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/index.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/index.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>Lucene Java Documentation</title>
<link type="text/css" href="skin/basic.css" rel="stylesheet">
Modified: lucene/dev/branches/flexscoring/lucene/docs/index.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/index.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/index.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/index.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/linkmap.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/linkmap.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/linkmap.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>Site Linkmap Table of Contents</title>
<link type="text/css" href="skin/basic.css" rel="stylesheet">
Modified: lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/linkmap.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Lucene Contrib
Modified: lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/lucene-contrib/index.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Query Parser Syntax
Modified: lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/queryparsersyntax.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/scoring.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/scoring.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/scoring.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/scoring.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>
Apache Lucene - Scoring
Modified: lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Files lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf (original) and lucene/dev/branches/flexscoring/lucene/docs/scoring.pdf Sat May 14 13:51:35 2011 differ
Modified: lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html (original)
+++ lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.html Sat May 14 13:51:35 2011
@@ -3,7 +3,7 @@
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-version" content="0.9">
<meta name="Forrest-skin-name" content="lucene">
<title>Apache Lucene - System Requirements</title>
<link type="text/css" href="skin/basic.css" rel="stylesheet">
Modified: lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/docs/systemrequirements.pdf?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/Document.java Sat May 14 13:51:35 2011
@@ -131,8 +131,13 @@ public final class Document {
/** Returns a field with the given name if any exist in this document, or
* null. If multiple fields exists with this name, this method returns the
* first value added.
- * Do not use this method with lazy loaded fields.
+ * Do not use this method with lazy loaded fields or {@link NumericField}.
+ * @deprecated use {@link #getFieldable} instead and cast depending on
+ * data type.
+ * @throws ClassCastException if you try to retrieve a numerical or
+ * lazy loaded field.
*/
+ @Deprecated
public final Field getField(String name) {
return (Field) getFieldable(name);
}
@@ -154,6 +159,8 @@ public final class Document {
* this document, or null. If multiple fields exist with this name, this
* method returns the first value added. If only binary fields with this name
* exist, returns null.
+ * For {@link NumericField} it returns the string value of the number. If you want
+ * the actual {@code NumericField} instance back, use {@link #getFieldable}.
*/
public final String get(String name) {
for (Fieldable field : fields) {
@@ -177,13 +184,18 @@ public final class Document {
/**
* Returns an array of {@link Field}s with the given name.
- * Do not use with lazy loaded fields.
* This method returns an empty array when there are no
* matching fields. It never returns null.
+ * Do not use this method with lazy loaded fields or {@link NumericField}.
*
* @param name the name of the field
* @return a <code>Field[]</code> array
+ * @deprecated use {@link #getFieldable} instead and cast depending on
+ * data type.
+ * @throws ClassCastException if you try to retrieve a numerical or
+ * lazy loaded field.
*/
+ @Deprecated
public final Field[] getFields(String name) {
List<Field> result = new ArrayList<Field>();
for (Fieldable field : fields) {
@@ -230,6 +242,8 @@ public final class Document {
* Returns an array of values of the field specified as the method parameter.
* This method returns an empty array when there are no
* matching fields. It never returns null.
+ * For {@link NumericField}s it returns the string value of the number. If you want
+ * the actual {@code NumericField} instances back, use {@link #getFieldables}.
* @param name the name of the field
* @return a <code>String[]</code> of field values
*/
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/document/NumericField.java Sat May 14 13:51:35 2011
@@ -127,18 +127,18 @@ import org.apache.lucene.search.FieldCac
* class is a wrapper around this token stream type for
* easier, more intuitive usage.</p>
*
- * <p><b>NOTE:</b> This class is only used during
- * indexing. When retrieving the stored field value from a
- * {@link Document} instance after search, you will get a
- * conventional {@link Fieldable} instance where the numeric
- * values are returned as {@link String}s (according to
- * <code>toString(value)</code> of the used data type).
- *
* @since 2.9
*/
public final class NumericField extends AbstractField {
- private final NumericTokenStream numericTS;
+ /** Data type of the value in {@link NumericField}.
+ * @since 3.2
+ */
+ public static enum DataType { INT, LONG, FLOAT, DOUBLE }
+
+ private transient NumericTokenStream numericTS;
+ private DataType type;
+ private final int precisionStep;
/**
* Creates a field for numeric values using the default <code>precisionStep</code>
@@ -158,8 +158,8 @@ public final class NumericField extends
* a numeric value, before indexing a document containing this field,
* set a value using the various set<em>???</em>Value() methods.
* @param name the field name
- * @param store if the field should be stored in plain text form
- * (according to <code>toString(value)</code> of the used data type)
+ * @param store if the field should be stored, {@link Document#getFieldable}
+ * then returns {@code NumericField} instances on search results.
* @param index if the field should be indexed using {@link NumericTokenStream}
*/
public NumericField(String name, Field.Store store, boolean index) {
@@ -186,19 +186,43 @@ public final class NumericField extends
* set a value using the various set<em>???</em>Value() methods.
* @param name the field name
* @param precisionStep the used <a href="../search/NumericRangeQuery.html#precisionStepDesc">precision step</a>
- * @param store if the field should be stored in plain text form
- * (according to <code>toString(value)</code> of the used data type)
+ * @param store if the field should be stored, {@link Document#getFieldable}
+ * then returns {@code NumericField} instances on search results.
* @param index if the field should be indexed using {@link NumericTokenStream}
*/
public NumericField(String name, int precisionStep, Field.Store store, boolean index) {
super(name, store, index ? Field.Index.ANALYZED_NO_NORMS : Field.Index.NO, Field.TermVector.NO);
+ this.precisionStep = precisionStep;
setOmitTermFreqAndPositions(true);
- numericTS = new NumericTokenStream(precisionStep);
}
/** Returns a {@link NumericTokenStream} for indexing the numeric value. */
public TokenStream tokenStreamValue() {
- return isIndexed() ? numericTS : null;
+ if (!isIndexed())
+ return null;
+ if (numericTS == null) {
+ // lazy init the TokenStream as it is heavy to instantiate (attributes,...),
+ // if not needed (stored field loading)
+ numericTS = new NumericTokenStream(precisionStep);
+ // initialize value in TokenStream
+ if (fieldsData != null) {
+ assert type != null;
+ final Number val = (Number) fieldsData;
+ switch (type) {
+ case INT:
+ numericTS.setIntValue(val.intValue()); break;
+ case LONG:
+ numericTS.setLongValue(val.longValue()); break;
+ case FLOAT:
+ numericTS.setFloatValue(val.floatValue()); break;
+ case DOUBLE:
+ numericTS.setDoubleValue(val.doubleValue()); break;
+ default:
+ assert false : "Should never get here";
+ }
+ }
+ }
+ return numericTS;
}
/** Returns always <code>null</code> for numeric fields */
@@ -212,7 +236,10 @@ public final class NumericField extends
return null;
}
- /** Returns the numeric value as a string (how it is stored, when {@link Field.Store#YES} is chosen). */
+ /** Returns the numeric value as a string. This format is also returned if you call {@link Document#get(String)}
+ * on search results. It is recommended to use {@link Document#getFieldable} instead
+ * that returns {@code NumericField} instances. You can then use {@link #getNumericValue}
+ * to return the stored value. */
public String stringValue() {
return (fieldsData == null) ? null : fieldsData.toString();
}
@@ -224,7 +251,14 @@ public final class NumericField extends
/** Returns the precision step. */
public int getPrecisionStep() {
- return numericTS.getPrecisionStep();
+ return precisionStep;
+ }
+
+ /** Returns the data type of the current value, {@code null} if not yet set.
+ * @since 3.2
+ */
+ public DataType getDataType() {
+ return type;
}
/**
@@ -234,8 +268,9 @@ public final class NumericField extends
* <code>document.add(new NumericField(name, precisionStep).setLongValue(value))</code>
*/
public NumericField setLongValue(final long value) {
- numericTS.setLongValue(value);
+ if (numericTS != null) numericTS.setLongValue(value);
fieldsData = Long.valueOf(value);
+ type = DataType.LONG;
return this;
}
@@ -246,8 +281,9 @@ public final class NumericField extends
* <code>document.add(new NumericField(name, precisionStep).setIntValue(value))</code>
*/
public NumericField setIntValue(final int value) {
- numericTS.setIntValue(value);
+ if (numericTS != null) numericTS.setIntValue(value);
fieldsData = Integer.valueOf(value);
+ type = DataType.INT;
return this;
}
@@ -258,8 +294,9 @@ public final class NumericField extends
* <code>document.add(new NumericField(name, precisionStep).setDoubleValue(value))</code>
*/
public NumericField setDoubleValue(final double value) {
- numericTS.setDoubleValue(value);
+ if (numericTS != null) numericTS.setDoubleValue(value);
fieldsData = Double.valueOf(value);
+ type = DataType.DOUBLE;
return this;
}
@@ -270,8 +307,9 @@ public final class NumericField extends
* <code>document.add(new NumericField(name, precisionStep).setFloatValue(value))</code>
*/
public NumericField setFloatValue(final float value) {
- numericTS.setFloatValue(value);
+ if (numericTS != null) numericTS.setFloatValue(value);
fieldsData = Float.valueOf(value);
+ type = DataType.FLOAT;
return this;
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java Sat May 14 13:51:35 2011
@@ -81,6 +81,6 @@ final class ByteSliceWriter extends Data
}
public int getAddress() {
- return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+ return upto + (offset0 & DocumentsWriterPerThread.BYTE_BLOCK_NOT_MASK);
}
}
\ No newline at end of file
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sat May 14 13:51:35 2011
@@ -661,10 +661,13 @@ public class CheckIndex {
status.termCount++;
final DocsEnum docs2;
+ final boolean hasPositions;
if (postings != null) {
docs2 = postings;
+ hasPositions = true;
} else {
docs2 = docs;
+ hasPositions = false;
}
int lastDoc = -1;
@@ -733,6 +736,67 @@ public class CheckIndex {
throw new RuntimeException("term " + term + " totalTermFreq=" + totalTermFreq2 + " != recomputed totalTermFreq=" + totalTermFreq);
}
}
+
+ // Test skipping
+ if (docFreq >= 16) {
+ if (hasPositions) {
+ for(int idx=0;idx<7;idx++) {
+ final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
+ postings = terms.docsAndPositions(delDocs, postings);
+ final int docID = postings.advance(skipDocID);
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ } else {
+ if (docID < skipDocID) {
+ throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID);
+ }
+ final int freq = postings.freq();
+ if (freq <= 0) {
+ throw new RuntimeException("termFreq " + freq + " is out of bounds");
+ }
+ int lastPosition = -1;
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ final int pos = postings.nextPosition();
+ if (pos < 0) {
+ throw new RuntimeException("position " + pos + " is out of bounds");
+ }
+ if (pos <= lastPosition) {
+ throw new RuntimeException("position " + pos + " is <= lastPosition " + lastPosition);
+ }
+ lastPosition = pos;
+ }
+
+ final int nextDocID = postings.nextDoc();
+ if (nextDocID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ if (nextDocID <= docID) {
+ throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID);
+ }
+ }
+ }
+ } else {
+ for(int idx=0;idx<7;idx++) {
+ final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
+ docs = terms.docs(delDocs, docs);
+ final int docID = docs.advance(skipDocID);
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ } else {
+ if (docID < skipDocID) {
+ throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID);
+ }
+ final int nextDocID = docs.nextDoc();
+ if (nextDocID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ if (nextDocID <= docID) {
+ throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID);
+ }
+ }
+ }
+ }
+ }
}
if (sumTotalTermFreq != 0) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java Sat May 14 13:51:35 2011
@@ -46,8 +46,10 @@ import org.apache.lucene.util.IOUtils;
* file. The {directory} that follows has that many entries. Each directory entry
* contains a long pointer to the start of this file's data section, and a String
* with that file's name.
+ *
+ * @lucene.internal
*/
-final class CompoundFileWriter {
+public final class CompoundFileWriter {
private static final class FileEntry {
/** source file */
@@ -136,8 +138,7 @@ final class CompoundFileWriter {
/** Merge files with the extensions added up to now.
* All files with these extensions are combined sequentially into the
- * compound stream. After successful merge, the source files
- * are deleted.
+ * compound stream.
* @throws IllegalStateException if close() had been called before or
* if no file has been added to this object
*/
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sat May 14 13:51:35 2011
@@ -135,8 +135,8 @@ public class ConcurrentMergeScheduler ex
final MergePolicy.OneMerge m1 = t1.getCurrentMerge();
final MergePolicy.OneMerge m2 = t2.getCurrentMerge();
- final int c1 = m1 == null ? Integer.MAX_VALUE : m1.segments.totalDocCount();
- final int c2 = m2 == null ? Integer.MAX_VALUE : m2.segments.totalDocCount();
+ final int c1 = m1 == null ? Integer.MAX_VALUE : m1.totalDocCount;
+ final int c2 = m2 == null ? Integer.MAX_VALUE : m2.totalDocCount;
return c2 - c1;
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocConsumer.java Sat May 14 13:51:35 2011
@@ -18,11 +18,12 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Collection;
abstract class DocConsumer {
- abstract DocConsumerPerThread addThread(DocumentsWriterThreadState perThread) throws IOException;
- abstract void flush(final Collection<DocConsumerPerThread> threads, final SegmentWriteState state) throws IOException;
+ abstract void processDocument(FieldInfos fieldInfos) throws IOException;
+ abstract void finishDocument() throws IOException;
+ abstract void flush(final SegmentWriteState state) throws IOException;
abstract void abort();
abstract boolean freeRAM();
+ abstract void doAfterFlush();
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java Sat May 14 13:51:35 2011
@@ -18,22 +18,25 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Collection;
import java.util.Map;
abstract class DocFieldConsumer {
- /** Called when DocumentsWriter decides to create a new
+ /** Called when DocumentsWriterPerThread decides to create a new
* segment */
- abstract void flush(Map<DocFieldConsumerPerThread,Collection<DocFieldConsumerPerField>> threadsAndFields, SegmentWriteState state) throws IOException;
+ abstract void flush(Map<FieldInfo, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException;
/** Called when an aborting exception is hit */
abstract void abort();
- /** Add a new thread */
- abstract DocFieldConsumerPerThread addThread(DocFieldProcessorPerThread docFieldProcessorPerThread) throws IOException;
-
- /** Called when DocumentsWriter is using too much RAM.
+ /** Called when DocumentsWriterPerThread is using too much RAM.
* The consumer should free RAM, if possible, returning
* true if any RAM was in fact freed. */
abstract boolean freeRAM();
- }
+
+ abstract void startDocument() throws IOException;
+
+ abstract DocFieldConsumerPerField addField(FieldInfo fi);
+
+ abstract void finishDocument() throws IOException;
+
+}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java?rev=1103112&r1=1103111&r2=1103112&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Sat May 14 13:51:35 2011
@@ -24,4 +24,5 @@ abstract class DocFieldConsumerPerField
/** Processes all occurrences of a single field */
abstract void processFields(Fieldable[] fields, int count) throws IOException;
abstract void abort();
+ abstract FieldInfo getFieldInfo();
}