You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/07/18 15:35:33 UTC
svn commit: r1147873 [8/13] - in /lucene/dev/branches/positions: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/lucene/contrib/memory/
dev-tools/idea/lucene/contrib/querypar...
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/fileformats.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
@@ -415,10 +412,14 @@ document.write("Last Published: " + docu
to stored fields file, previously they were stored in
text format only.
</p>
+<p>
+ In version 3.4, fields can omit position data while
+ still indexing term frequencies.
+ </p>
</div>
-<a name="N1003A"></a><a name="Definitions"></a>
+<a name="N1003D"></a><a name="Definitions"></a>
<h2 class="boxed">Definitions</h2>
<div class="section">
<p>
@@ -459,7 +460,7 @@ document.write("Last Published: " + docu
strings, the first naming the field, and the second naming text
within the field.
</p>
-<a name="N1005A"></a><a name="Inverted Indexing"></a>
+<a name="N1005D"></a><a name="Inverted Indexing"></a>
<h3 class="boxed">Inverted Indexing</h3>
<p>
The index stores statistics about terms in order
@@ -469,7 +470,7 @@ document.write("Last Published: " + docu
it. This is the inverse of the natural relationship, in which
documents list terms.
</p>
-<a name="N10066"></a><a name="Types of Fields"></a>
+<a name="N10069"></a><a name="Types of Fields"></a>
<h3 class="boxed">Types of Fields</h3>
<p>
In Lucene, fields may be <i>stored</i>, in which
@@ -483,7 +484,7 @@ document.write("Last Published: " + docu
to be indexed literally.
</p>
<p>See the <a href="api/core/org/apache/lucene/document/Field.html">Field</a> java docs for more information on Fields.</p>
-<a name="N10083"></a><a name="Segments"></a>
+<a name="N10086"></a><a name="Segments"></a>
<h3 class="boxed">Segments</h3>
<p>
Lucene indexes may be composed of multiple sub-indexes, or
@@ -509,7 +510,7 @@ document.write("Last Published: " + docu
Searches may involve multiple segments and/or multiple indexes, each
index potentially composed of a set of segments.
</p>
-<a name="N100A1"></a><a name="Document Numbers"></a>
+<a name="N100A4"></a><a name="Document Numbers"></a>
<h3 class="boxed">Document Numbers</h3>
<p>
Internally, Lucene refers to documents by an integer <i>document
@@ -564,7 +565,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N100C8"></a><a name="Overview"></a>
+<a name="N100CB"></a><a name="Overview"></a>
<h2 class="boxed">Overview</h2>
<div class="section">
<p>
@@ -611,7 +612,7 @@ document.write("Last Published: " + docu
<p>Term Frequency
data. For each term in the dictionary, the numbers of all the
documents that contain that term, and the frequency of the term in
- that document if omitTf is false.
+ that document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
</p>
</li>
@@ -622,8 +623,7 @@ document.write("Last Published: " + docu
<p>Term Proximity
data. For each term in the dictionary, the positions that the term
occurs in each document. Note that this will
- not exist if all fields in all documents set
- omitTf to true.
+ not exist if all fields in all documents omit position data.
</p>
</li>
@@ -663,7 +663,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N1010B"></a><a name="File Naming"></a>
+<a name="N1010E"></a><a name="File Naming"></a>
<h2 class="boxed">File Naming</h2>
<div class="section">
<p>
@@ -690,7 +690,7 @@ document.write("Last Published: " + docu
</p>
</div>
-<a name="N1011A"></a><a name="file-names"></a>
+<a name="N1011D"></a><a name="file-names"></a>
<h2 class="boxed">Summary of File Extensions</h2>
<div class="section">
<p>The following table summarizes the names and extensions of the files in Lucene:
@@ -840,10 +840,10 @@ document.write("Last Published: " + docu
</div>
-<a name="N10212"></a><a name="Primitive Types"></a>
+<a name="N10215"></a><a name="Primitive Types"></a>
<h2 class="boxed">Primitive Types</h2>
<div class="section">
-<a name="N10217"></a><a name="Byte"></a>
+<a name="N1021A"></a><a name="Byte"></a>
<h3 class="boxed">Byte</h3>
<p>
The most primitive type
@@ -851,7 +851,7 @@ document.write("Last Published: " + docu
other data types are defined as sequences
of bytes, so file formats are byte-order independent.
</p>
-<a name="N10220"></a><a name="UInt32"></a>
+<a name="N10223"></a><a name="UInt32"></a>
<h3 class="boxed">UInt32</h3>
<p>
32-bit unsigned integers are written as four
@@ -861,7 +861,7 @@ document.write("Last Published: " + docu
UInt32 --> <Byte><sup>4</sup>
</p>
-<a name="N1022F"></a><a name="Uint64"></a>
+<a name="N10232"></a><a name="Uint64"></a>
<h3 class="boxed">Uint64</h3>
<p>
64-bit unsigned integers are written as eight
@@ -870,7 +870,7 @@ document.write("Last Published: " + docu
<p>UInt64 --> <Byte><sup>8</sup>
</p>
-<a name="N1023E"></a><a name="VInt"></a>
+<a name="N10241"></a><a name="VInt"></a>
<h3 class="boxed">VInt</h3>
<p>
A variable-length format for positive integers is
@@ -1420,13 +1420,13 @@ document.write("Last Published: " + docu
This provides compression while still being
efficient to decode.
</p>
-<a name="N10523"></a><a name="Chars"></a>
+<a name="N10526"></a><a name="Chars"></a>
<h3 class="boxed">Chars</h3>
<p>
Lucene writes unicode
character sequences as UTF-8 encoded bytes.
</p>
-<a name="N1052C"></a><a name="String"></a>
+<a name="N1052F"></a><a name="String"></a>
<h3 class="boxed">String</h3>
<p>
Lucene writes strings as UTF-8 encoded bytes.
@@ -1439,10 +1439,10 @@ document.write("Last Published: " + docu
</div>
-<a name="N10539"></a><a name="Compound Types"></a>
+<a name="N1053C"></a><a name="Compound Types"></a>
<h2 class="boxed">Compound Types</h2>
<div class="section">
-<a name="N1053E"></a><a name="MapStringString"></a>
+<a name="N10541"></a><a name="MapStringString"></a>
<h3 class="boxed">Map<String,String></h3>
<p>
In a couple places Lucene stores a Map
@@ -1455,13 +1455,13 @@ document.write("Last Published: " + docu
</div>
-<a name="N1054E"></a><a name="Per-Index Files"></a>
+<a name="N10551"></a><a name="Per-Index Files"></a>
<h2 class="boxed">Per-Index Files</h2>
<div class="section">
<p>
The files in this section exist one-per-index.
</p>
-<a name="N10556"></a><a name="Segments File"></a>
+<a name="N10559"></a><a name="Segments File"></a>
<h3 class="boxed">Segments File</h3>
<p>
The active segments in the index are stored in the
@@ -1616,7 +1616,7 @@ document.write("Last Published: " + docu
</p>
<p>
HasProx is 1 if any fields in this segment have
- omitTf set to false; else, it's 0.
+ position data (IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); else, it's 0.
</p>
<p>
CommitUserData stores an optional user-supplied
@@ -1634,7 +1634,7 @@ document.write("Last Published: " + docu
<p> HasVectors is 1 if this segment stores term vectors,
else it's 0.
</p>
-<a name="N105E1"></a><a name="Lock File"></a>
+<a name="N105E4"></a><a name="Lock File"></a>
<h3 class="boxed">Lock File</h3>
<p>
The write lock, which is stored in the index
@@ -1648,14 +1648,14 @@ document.write("Last Published: " + docu
documents). This lock file ensures that only one
writer is modifying the index at a time.
</p>
-<a name="N105EA"></a><a name="Deletable File"></a>
+<a name="N105ED"></a><a name="Deletable File"></a>
<h3 class="boxed">Deletable File</h3>
<p>
A writer dynamically computes
the files that are deletable, instead, so no file
is written.
</p>
-<a name="N105F3"></a><a name="Compound Files"></a>
+<a name="N105F6"></a><a name="Compound Files"></a>
<h3 class="boxed">Compound Files</h3>
<p>Starting with Lucene 1.4 the compound file format became default. This
is simply a container for all files described in the next section
@@ -1684,14 +1684,14 @@ document.write("Last Published: " + docu
</div>
-<a name="N10624"></a><a name="Per-Segment Files"></a>
+<a name="N10627"></a><a name="Per-Segment Files"></a>
<h2 class="boxed">Per-Segment Files</h2>
<div class="section">
<p>
The remaining files are all per-segment, and are
thus defined by suffix.
</p>
-<a name="N1062C"></a><a name="Fields"></a>
+<a name="N1062F"></a><a name="Fields"></a>
<h3 class="boxed">Fields</h3>
<p>
@@ -1744,12 +1744,16 @@ document.write("Last Published: " + docu
<li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li>
<li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li>
+
+<li>If the seventh lowest-order bit is set (0x40), term frequencies and positions omitted for the indexed field.</li>
+
+<li>If the eighth lowest-order bit is set (0x80), positions are omitted for the indexed field.</li>
</ul>
</p>
<p>
- FNMVersion (added in 2.9) is always -2.
+ FNMVersion (added in 2.9) is -2 for indexes from 2.9 - 3.3. It is -3 for indexes in Lucene 3.4+
</p>
<p>
Fields are numbered by their order in this file. Thus field zero is
@@ -1901,7 +1905,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N106E7"></a><a name="Term Dictionary"></a>
+<a name="N106F0"></a><a name="Term Dictionary"></a>
<h3 class="boxed">Term Dictionary</h3>
<p>
The term dictionary is represented as two files:
@@ -2005,7 +2009,7 @@ document.write("Last Published: " + docu
file. In particular, it is the difference between the position of
this term's data in that file and the position of the previous
term's data (or zero, for the first term in the file. For fields
- with omitTf true, this will be 0 since
+ that omit position data, this will be 0 since
prox information is not stored.
</p>
@@ -2093,12 +2097,12 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N1076B"></a><a name="Frequencies"></a>
+<a name="N10774"></a><a name="Frequencies"></a>
<h3 class="boxed">Frequencies</h3>
<p>
The .frq file contains the lists of documents
which contain each term, along with the frequency of the term in that
- document (if omitTf is false).
+ document (except when frequencies are omitted: IndexOptions.DOCS_ONLY).
</p>
<p>FreqFile (.frq) -->
<TermFreqs, SkipData>
@@ -2138,26 +2142,26 @@ document.write("Last Published: " + docu
<p>TermFreq
entries are ordered by increasing document number.
</p>
-<p>DocDelta: if omitTf is false, this determines both
+<p>DocDelta: if frequencies are indexed, this determines both
the document number and the frequency. In
particular, DocDelta/2 is the difference between
this document number and the previous document
number (or zero when this is the first document in
a TermFreqs). When DocDelta is odd, the frequency
is one. When DocDelta is even, the frequency is
- read as another VInt. If omitTf is true, DocDelta
+ read as another VInt. If frequencies are omitted, DocDelta
contains the gap (not multiplied by 2) between
document numbers and no frequency information is
stored.
</p>
<p>For example, the TermFreqs for a term which occurs
once in document seven and three times in document
- eleven, with omitTf false, would be the following
+ eleven, with frequencies indexed, would be the following
sequence of VInts:
</p>
<p>15, 8, 3
</p>
-<p> If omitTf were true it would be this sequence
+<p> If frequencies were omitted (IndexOptions.DOCS_ONLY) it would be this sequence
of VInts instead:
</p>
<p>
@@ -2221,14 +2225,14 @@ document.write("Last Published: " + docu
entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
to entry 31 on level 0.
</p>
-<a name="N107F3"></a><a name="Positions"></a>
+<a name="N107FC"></a><a name="Positions"></a>
<h3 class="boxed">Positions</h3>
<p>
The .prx file contains the lists of positions that
each term occurs at within documents. Note that
- fields with omitTf true do not store
+ fields omitting positional data do not store
anything into this file, and if all fields in the
- index have omitTf true then the .prx file will not
+ index omit positional data then the .prx file will not
exist.
</p>
<p>ProxFile (.prx) -->
@@ -2291,7 +2295,7 @@ document.write("Last Published: " + docu
Payload. If PayloadLength is not stored, then this Payload has the same
length as the Payload at the previous position.
</p>
-<a name="N1082F"></a><a name="Normalization Factors"></a>
+<a name="N10838"></a><a name="Normalization Factors"></a>
<h3 class="boxed">Normalization Factors</h3>
<p>There's a single .nrm file containing all norms:
</p>
@@ -2371,7 +2375,7 @@ document.write("Last Published: " + docu
</p>
<p>Separate norm files are created (when adequate) for both compound and non compound segments.
</p>
-<a name="N10880"></a><a name="Term Vectors"></a>
+<a name="N10889"></a><a name="Term Vectors"></a>
<h3 class="boxed">Term Vectors</h3>
<p>
Term Vector support is an optional on a field by
@@ -2507,7 +2511,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N1091C"></a><a name="Deleted Documents"></a>
+<a name="N10925"></a><a name="Deleted Documents"></a>
<h3 class="boxed">Deleted Documents</h3>
<p>The .del file is
optional, and only exists when a segment contains deletions.
@@ -2571,7 +2575,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N10956"></a><a name="Limitations"></a>
+<a name="N1095F"></a><a name="Limitations"></a>
<h2 class="boxed">Limitations</h2>
<div class="section">
<p>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/gettingstarted.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/index.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/index.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/index.html Mon Jul 18 13:34:29 2011
@@ -166,9 +166,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/linkmap.html Mon Jul 18 13:34:29 2011
@@ -166,9 +166,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
@@ -358,12 +355,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a> ___________________ <em>javadoc-contrib-spellchecker</em>
</li>
</ul>
-
-<ul>
-<li>
-<a href="api/contrib-wordnet/index.html">Wordnet</a> ___________________ <em>javadoc-contrib-wordnet</em>
-</li>
-</ul>
<ul>
<li>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/lucene-contrib/index.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
<a href="../api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="../api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="../api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
@@ -263,9 +260,6 @@ document.write("Last Published: " + docu
<a href="#spellchecker">spellchecker</a>
</li>
<li>
-<a href="#wordnet">wordnet</a>
-</li>
-<li>
<a href="#xml-query-parser">xml-query-parser</a>
</li>
</ul>
@@ -375,15 +369,10 @@ document.write("Last Published: " + docu
<p>Provides tools for spellchecking and suggestions with Lucene.</p>
<p>See <a href="../api/contrib-spellchecker/index.html">spellchecker javadoc</a>
</p>
-<a name="N100DE"></a><a name="wordnet"></a>
-<h3 class="boxed">wordnet</h3>
-<p>Tools to help utilize wordnet synonyms with Lucene</p>
-<p>See <a href="../api/contrib-wordnet/index.html">wordnet javadoc</a>
-</p>
-<a name="N100ED"></a><a name="xml-query-parser"></a>
+<a name="N100DE"></a><a name="xml-query-parser"></a>
<h3 class="boxed">xml-query-parser</h3>
<p>A QueryParser that can read queries written in an XML format.</p>
-<p>See <a href="../api/contrib-wordnet/index.html">xml-query-parser javadoc</a>
+<p>See <a href="../api/contrib-xml-query-parser/index.html">xml-query-parser javadoc</a>
</p>
</div>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/queryparsersyntax.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/scoring.html Mon Jul 18 13:34:29 2011
@@ -168,9 +168,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
Modified: lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html (original)
+++ lucene/dev/branches/positions/lucene/src/site/build/site/systemrequirements.html Mon Jul 18 13:34:29 2011
@@ -166,9 +166,6 @@ document.write("Last Published: " + docu
<a href="api/contrib-spellchecker/index.html">Spellchecker</a>
</div>
<div class="menuitem">
-<a href="api/contrib-wordnet/index.html">Wordnet</a>
-</div>
-<div class="menuitem">
<a href="api/contrib-xml-query-parser/index.html">XML Query Parser</a>
</div>
</div>
Modified: lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml (original)
+++ lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/fileformats.xml Mon Jul 18 13:34:29 2011
@@ -99,6 +99,10 @@
to stored fields file, previously they were stored in
text format only.
</p>
+ <p>
+ In version 3.4, fields can omit position data while
+ still indexing term frequencies.
+ </p>
</section>
<section id="Definitions"><title>Definitions</title>
@@ -276,7 +280,7 @@
<p>Term Frequency
data. For each term in the dictionary, the numbers of all the
documents that contain that term, and the frequency of the term in
- that document if omitTf is false.
+ that document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
</p>
</li>
@@ -284,8 +288,7 @@
<p>Term Proximity
data. For each term in the dictionary, the positions that the term
occurs in each document. Note that this will
- not exist if all fields in all documents set
- omitTf to true.
+ not exist if all fields in all documents omit position data.
</p>
</li>
@@ -1080,7 +1083,7 @@
<p>
HasProx is 1 if any fields in this segment have
- omitTf set to false; else, it's 0.
+ position data (IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); else, it's 0.
</p>
<p>
@@ -1217,11 +1220,13 @@
<li>If the fourth lowest-order bit is set (0x08), term offsets are stored with the term vectors.</li>
<li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li>
<li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li>
+ <li>If the seventh lowest-order bit is set (0x40), term frequencies and positions omitted for the indexed field.</li>
+ <li>If the eighth lowest-order bit is set (0x80), positions are omitted for the indexed field.</li>
</ul>
</p>
<p>
- FNMVersion (added in 2.9) is always -2.
+ FNMVersion (added in 2.9) is -2 for indexes from 2.9 - 3.3. It is -3 for indexes in Lucene 3.4+
</p>
<p>
@@ -1419,7 +1424,7 @@
file. In particular, it is the difference between the position of
this term's data in that file and the position of the previous
term's data (or zero, for the first term in the file. For fields
- with omitTf true, this will be 0 since
+ that omit position data, this will be 0 since
prox information is not stored.
</p>
<p>SkipDelta determines the position of this
@@ -1494,7 +1499,7 @@
<p>
The .frq file contains the lists of documents
which contain each term, along with the frequency of the term in that
- document (if omitTf is false).
+ document (except when frequencies are omitted: IndexOptions.DOCS_ONLY).
</p>
<p>FreqFile (.frq) -->
<TermFreqs, SkipData>
@@ -1531,26 +1536,26 @@
<p>TermFreq
entries are ordered by increasing document number.
</p>
- <p>DocDelta: if omitTf is false, this determines both
+ <p>DocDelta: if frequencies are indexed, this determines both
the document number and the frequency. In
particular, DocDelta/2 is the difference between
this document number and the previous document
number (or zero when this is the first document in
a TermFreqs). When DocDelta is odd, the frequency
is one. When DocDelta is even, the frequency is
- read as another VInt. If omitTf is true, DocDelta
+ read as another VInt. If frequencies are omitted, DocDelta
contains the gap (not multiplied by 2) between
document numbers and no frequency information is
stored.
</p>
<p>For example, the TermFreqs for a term which occurs
once in document seven and three times in document
- eleven, with omitTf false, would be the following
+ eleven, with frequencies indexed, would be the following
sequence of VInts:
</p>
<p>15, 8, 3
</p>
- <p> If omitTf were true it would be this sequence
+ <p> If frequencies were omitted (IndexOptions.DOCS_ONLY) it would be this sequence
of VInts instead:
</p>
<p>
@@ -1621,9 +1626,9 @@
<p>
The .prx file contains the lists of positions that
each term occurs at within documents. Note that
- fields with omitTf true do not store
+ fields omitting positional data do not store
anything into this file, and if all fields in the
- index have omitTf true then the .prx file will not
+ index omit positional data then the .prx file will not
exist.
</p>
<p>ProxFile (.prx) -->
Modified: lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml (original)
+++ lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/lucene-contrib/index.xml Mon Jul 18 13:34:29 2011
@@ -106,14 +106,9 @@
<p>See <a href="../api/contrib-spellchecker/index.html">spellchecker javadoc</a></p>
</section>
- <section id="wordnet"><title>wordnet</title>
- <p>Tools to help utilize wordnet synonyms with Lucene</p>
- <p>See <a href="../api/contrib-wordnet/index.html">wordnet javadoc</a></p>
- </section>
-
<section id="xml-query-parser"><title>xml-query-parser</title>
<p>A QueryParser that can read queries written in an XML format.</p>
- <p>See <a href="../api/contrib-wordnet/index.html">xml-query-parser javadoc</a></p>
+ <p>See <a href="../api/contrib-xml-query-parser/index.html">xml-query-parser javadoc</a></p>
</section>
</section>
Modified: lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml (original)
+++ lucene/dev/branches/positions/lucene/src/site/src/documentation/content/xdocs/site.xml Mon Jul 18 13:34:29 2011
@@ -66,7 +66,6 @@ See http://forrest.apache.org/docs/linki
<javadoc-contrib-remote label="Remote" href="ext:javadocs-contrib-remote"/>
<javadoc-contrib-spatial label="Spatial" href="ext:javadocs-contrib-spatial"/>
<javadoc-contrib-spellchecker label="Spellchecker" href="ext:javadocs-contrib-spellchecker"/>
- <javadoc-contrib-wordnet label="Wordnet" href="ext:javadocs-contrib-wordnet"/>
<javadoc-contrib-xml-query-parser label="XML Query Parser" href="ext:javadocs-contrib-xml-query-parser"/>
</javadoc-contrib>
</javadoc>
@@ -106,7 +105,6 @@ See http://forrest.apache.org/docs/linki
<javadocs-contrib-remote href="api/contrib-remote/index.html"/>
<javadocs-contrib-spatial href="api/contrib-spatial/index.html"/>
<javadocs-contrib-spellchecker href="api/contrib-spellchecker/index.html"/>
- <javadocs-contrib-wordnet href="api/contrib-wordnet/index.html"/>
<javadocs-contrib-xml-query-parser href="api/contrib-xml-query-parser/index.html"/>
<forrest href="http://forrest.apache.org/">
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Mon Jul 18 13:34:29 2011
@@ -260,7 +260,11 @@ public abstract class BaseTokenStreamTes
default:
text = _TestUtil.randomUnicodeString(random, maxWordLength);
}
-
+
+ if (VERBOSE) {
+ System.out.println("NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text);
+ }
+
TokenStream ts = a.reusableTokenStream("dummy", new StringReader(text));
assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class));
CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
@@ -286,6 +290,9 @@ public abstract class BaseTokenStreamTes
ts.close();
// verify reusing is "reproducable" and also get the normal tokenstream sanity checks
if (!tokens.isEmpty()) {
+ if (VERBOSE) {
+ System.out.println("NOTE: BaseTokenStreamTestCase: re-run analysis");
+ }
if (typeAtt != null && posIncAtt != null && offsetAtt != null) {
// offset + pos + type
assertAnalyzesToReuse(a, text,
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java Mon Jul 18 13:34:29 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -67,7 +68,7 @@ class DocHelper {
public static Field noTFField = new Field(NO_TF_KEY, NO_TF_TEXT,
Field.Store.YES, Field.Index.ANALYZED);
static {
- noTFField.setOmitTermFreqAndPositions(true);
+ noTFField.setIndexOptions(IndexOptions.DOCS_ONLY);
}
public static final String UNINDEXED_FIELD_TEXT = "unindexed field text";
@@ -173,7 +174,7 @@ class DocHelper {
if (f.isStored()) add(stored,f);
else add(unstored,f);
if (f.getOmitNorms()) add(noNorms,f);
- if (f.getOmitTermFreqAndPositions()) add(noTf,f);
+ if (f.getIndexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f);
if (f.isLazy()) add(lazy, f);
}
}
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java Mon Jul 18 13:34:29 2011
@@ -22,6 +22,7 @@ import java.util.Comparator;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.codecs.FieldsConsumer;
@@ -90,7 +91,7 @@ class PreFlexFieldsWriter extends Fields
public PreFlexTermsWriter(FieldInfo fieldInfo) {
this.fieldInfo = fieldInfo;
- omitTF = fieldInfo.omitTermFreqAndPositions;
+ omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
storePayloads = fieldInfo.storePayloads;
}
@@ -195,7 +196,7 @@ class PreFlexFieldsWriter extends Fields
}
@Override
- public void finish(long sumTotalTermCount) throws IOException {
+ public void finish(long sumTotalTermCount, long sumDocFreq) throws IOException {
}
@Override
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Mon Jul 18 13:34:29 2011
@@ -352,6 +352,7 @@ public abstract class LuceneTestCase ext
state = State.INITIAL;
staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : TwoLongs.fromString(TEST_SEED).l1;
random.setSeed(staticSeed);
+ random.initialized = true;
tempDirs.clear();
stores = Collections.synchronizedMap(new IdentityHashMap<MockDirectoryWrapper,StackTraceElement[]>());
@@ -494,6 +495,8 @@ public abstract class LuceneTestCase ext
}
}
}
+ random.setSeed(0L);
+ random.initialized = false;
}
private static boolean testsFailed; /* true if any tests failed */
@@ -1366,7 +1369,26 @@ public abstract class LuceneTestCase ext
private long seed;
private static final Random seedRand = new Random();
- protected static final Random random = new Random(0);
+ protected static final SmartRandom random = new SmartRandom(0);
+
+ public static class SmartRandom extends Random {
+ boolean initialized;
+
+ SmartRandom(long seed) {
+ super(seed);
+ }
+
+ @Override
+ protected int next(int bits) {
+ if (!initialized) {
+ System.err.println("!!! WARNING: test is using random from static initializer !!!");
+ Thread.dumpStack();
+ // I wish, but it causes JRE crashes
+ // throw new IllegalStateException("you cannot use this random from a static initializer in your test");
+ }
+ return super.next(bits);
+ }
+ }
private String name = "<unknown>";
Modified: lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/positions/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Mon Jul 18 13:34:29 2011
@@ -422,7 +422,7 @@ public class _TestUtil {
List<Fieldable> fields = doc.getFields();
for (Fieldable field : fields) {
fieldInfos.addOrUpdate(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
- field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions(), field.docValuesType());
+ field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getIndexOptions(), field.docValuesType());
}
}
@@ -507,7 +507,7 @@ public class _TestUtil {
field1.isStored() ? Field.Store.YES : Field.Store.NO,
field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
field2.setOmitNorms(field1.getOmitNorms());
- field2.setOmitTermFreqAndPositions(field1.getOmitTermFreqAndPositions());
+ field2.setIndexOptions(field1.getIndexOptions());
doc2.add(field2);
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Mon Jul 18 13:34:29 2011
@@ -102,6 +102,7 @@ public class TestExternalCodecs extends
final String field;
final SortedMap<String,RAMTerm> termToDocs = new TreeMap<String,RAMTerm>();
long sumTotalTermFreq;
+ long sumDocFreq;
RAMField(String field) {
this.field = field;
@@ -116,6 +117,11 @@ public class TestExternalCodecs extends
public long getSumTotalTermFreq() {
return sumTotalTermFreq;
}
+
+ @Override
+ public long getSumDocFreq() throws IOException {
+ return sumDocFreq;
+ }
@Override
public TermsEnum iterator() {
@@ -204,8 +210,9 @@ public class TestExternalCodecs extends
}
@Override
- public void finish(long sumTotalTermFreq) {
+ public void finish(long sumTotalTermFreq, long sumDocFreq) {
field.sumTotalTermFreq = sumTotalTermFreq;
+ field.sumDocFreq = sumDocFreq;
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/Test2BTerms.java Mon Jul 18 13:34:29 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.search.*;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.document.*;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.codecs.CodecProvider;
import java.io.File;
import java.io.IOException;
@@ -177,7 +178,7 @@ public class Test2BTerms extends LuceneT
Document doc = new Document();
final MyTokenStream ts = new MyTokenStream(random, TERMS_PER_DOC);
Field field = new Field("field", ts);
- field.setOmitTermFreqAndPositions(true);
+ field.setIndexOptions(IndexOptions.DOCS_ONLY);
field.setOmitNorms(true);
doc.add(field);
//w.setInfoStream(System.out);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Mon Jul 18 13:34:29 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.NumericField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.DocIdSetIterator;
@@ -95,6 +96,8 @@ public class TestBackwardsCompatibility
"31.nocfs",
"32.cfs",
"32.nocfs",
+ "34.cfs",
+ "34.nocfs",
};
final String[] unsupportedNames = {"19.cfs",
@@ -455,9 +458,12 @@ public class TestBackwardsCompatibility
File indexDir = new File(LuceneTestCase.TEMP_DIR, dirName);
_TestUtil.rmDir(indexDir);
Directory dir = newFSDirectory(indexDir);
-
- IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
+ LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
+ mp.setUseCompoundFile(doCFS);
+ mp.setNoCFSRatio(1.0);
+ // TODO: remove randomness
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMaxBufferedDocs(10).setMergePolicy(mp);
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<35;i++) {
@@ -471,8 +477,12 @@ public class TestBackwardsCompatibility
if (!optimized) {
// open fresh writer so we get no prx file in the added segment
- conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
+ mp = new LogByteSizeMergePolicy();
+ mp.setUseCompoundFile(doCFS);
+ mp.setNoCFSRatio(1.0);
+ // TODO: remove randomness
+ conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMaxBufferedDocs(10).setMergePolicy(mp);
writer = new IndexWriter(dir, conf);
addNoProxDoc(writer);
writer.close();
@@ -599,10 +609,10 @@ public class TestBackwardsCompatibility
private void addNoProxDoc(IndexWriter writer) throws IOException {
Document doc = new Document();
Field f = new Field("content3", "aaa", Field.Store.YES, Field.Index.ANALYZED);
- f.setOmitTermFreqAndPositions(true);
+ f.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(f);
f = new Field("content4", "aaa", Field.Store.YES, Field.Index.NO);
- f.setOmitTermFreqAndPositions(true);
+ f.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(f);
writer.addDocument(doc);
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestCodecs.java Mon Jul 18 13:34:29 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
@@ -45,6 +46,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
+import org.junit.BeforeClass;
// TODO: test multiple codecs here?
@@ -66,13 +68,18 @@ import org.apache.lucene.util._TestUtil;
public class TestCodecs extends LuceneTestCase {
private static String[] fieldNames = new String[] {"one", "two", "three", "four"};
- private final static int NUM_TEST_ITER = atLeast(20);
+ private static int NUM_TEST_ITER;
private final static int NUM_TEST_THREADS = 3;
private final static int NUM_FIELDS = 4;
private final static int NUM_TERMS_RAND = 50; // must be > 16 to test skipping
private final static int DOC_FREQ_RAND = 500; // must be > 16 to test skipping
private final static int TERM_DOC_FREQ_RAND = 20;
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ NUM_TEST_ITER = atLeast(20);
+ }
+
class FieldData implements Comparable {
final FieldInfo fieldInfo;
final TermData[] terms;
@@ -84,7 +91,8 @@ public class TestCodecs extends LuceneTe
this.storePayloads = storePayloads;
fieldInfos.addOrUpdate(name, true);
fieldInfo = fieldInfos.fieldInfo(name);
- fieldInfo.omitTermFreqAndPositions = omitTF;
+ // TODO: change this test to use all three
+ fieldInfo.indexOptions = omitTF ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
fieldInfo.storePayloads = storePayloads;
this.terms = terms;
for(int i=0;i<terms.length;i++)
@@ -101,10 +109,12 @@ public class TestCodecs extends LuceneTe
Arrays.sort(terms);
final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
long sumTotalTermCount = 0;
+ long sumDF = 0;
for (final TermData term : terms) {
+ sumDF += term.docs.length;
sumTotalTermCount += term.write(termsConsumer);
}
- termsConsumer.finish(sumTotalTermCount);
+ termsConsumer.finish(sumTotalTermCount, sumDF);
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon Jul 18 13:34:29 2011
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Field.
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IOContext.Context;
@@ -303,7 +304,7 @@ public class TestDocumentWriter extends
doc.add(newField("f1", "v2", Store.YES, Index.NO));
// f2 has no TF
Field f = newField("f2", "v1", Store.NO, Index.ANALYZED);
- f.setOmitTermFreqAndPositions(true);
+ f.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(f);
doc.add(newField("f2", "v2", Store.YES, Index.NO));
@@ -319,10 +320,10 @@ public class TestDocumentWriter extends
FieldInfos fi = reader.fieldInfos();
// f1
assertFalse("f1 should have no norms", reader.hasNorms("f1"));
- assertFalse("omitTermFreqAndPositions field bit should not be set for f1", fi.fieldInfo("f1").omitTermFreqAndPositions);
+ assertEquals("omitTermFreqAndPositions field bit should not be set for f1", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
// f2
assertTrue("f2 should have norms", reader.hasNorms("f2"));
- assertTrue("omitTermFreqAndPositions field bit should be set for f2", fi.fieldInfo("f2").omitTermFreqAndPositions);
+ assertEquals("omitTermFreqAndPositions field bit should be set for f2", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
reader.close();
}
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Mon Jul 18 13:34:29 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
@@ -137,7 +138,7 @@ public class TestFieldInfos extends Luce
try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
- random.nextBoolean(), random.nextBoolean(), null);
+ random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
fail("instance should be read only");
} catch (IllegalStateException e) {
// expected
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Mon Jul 18 13:34:29 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.document.FieldS
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.LoadFirstFieldSelector;
import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.AlreadyClosedException;
@@ -91,7 +92,7 @@ public class TestFieldsReader extends Lu
assertTrue(field.isStoreOffsetWithTermVector() == true);
assertTrue(field.isStorePositionWithTermVector() == true);
assertTrue(field.getOmitNorms() == false);
- assertTrue(field.getOmitTermFreqAndPositions() == false);
+ assertTrue(field.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
field = doc.getField(DocHelper.TEXT_FIELD_3_KEY);
assertTrue(field != null);
@@ -99,7 +100,7 @@ public class TestFieldsReader extends Lu
assertTrue(field.isStoreOffsetWithTermVector() == false);
assertTrue(field.isStorePositionWithTermVector() == false);
assertTrue(field.getOmitNorms() == true);
- assertTrue(field.getOmitTermFreqAndPositions() == false);
+ assertTrue(field.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
field = doc.getField(DocHelper.NO_TF_KEY);
assertTrue(field != null);
@@ -107,7 +108,7 @@ public class TestFieldsReader extends Lu
assertTrue(field.isStoreOffsetWithTermVector() == false);
assertTrue(field.isStorePositionWithTermVector() == false);
assertTrue(field.getOmitNorms() == false);
- assertTrue(field.getOmitTermFreqAndPositions() == true);
+ assertTrue(field.getIndexOptions() == IndexOptions.DOCS_ONLY);
reader.close();
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Mon Jul 18 13:34:29 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
@@ -166,6 +167,13 @@ public class TestIndexWriterCommit exten
* measure max temp disk space used.
*/
public void testCommitOnCloseDiskUsage() throws IOException {
+ // MemoryCodec, since it uses FST, is not necessarily
+ // "additive", ie if you add up N small FSTs, then merge
+ // them, the merged result can easily be larger than the
+ // sum because the merged FST may use array encoding for
+ // some arcs (which uses more space):
+ assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory"));
+ assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory"));
MockDirectoryWrapper dir = newDirectory();
Analyzer analyzer;
if (random.nextBoolean()) {
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Mon Jul 18 13:34:29 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
@@ -142,6 +143,14 @@ public class TestIndexWriterOnDiskFull e
*/
public void testAddIndexOnDiskFull() throws IOException
{
+ // MemoryCodec, since it uses FST, is not necessarily
+ // "additive", ie if you add up N small FSTs, then merge
+ // them, the merged result can easily be larger than the
+ // sum because the merged FST may use array encoding for
+ // some arcs (which uses more space):
+ assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory"));
+ assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory"));
+
int START_COUNT = 57;
int NUM_DIR = TEST_NIGHTLY ? 50 : 5;
int END_COUNT = START_COUNT + NUM_DIR* (TEST_NIGHTLY ? 25 : 5);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Mon Jul 18 13:34:29 2011
@@ -26,11 +26,12 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util._TestUtil;
public class TestLongPostings extends LuceneTestCase {
@@ -63,8 +64,8 @@ public class TestLongPostings extends Lu
}
public void testLongPostings() throws Exception {
- assumeFalse("Too slow with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
- assumeFalse("Too slow with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
+ assumeFalse("Too slow with SimpleText codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
+ assumeFalse("Too slow with Memory codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
// Don't use _TestUtil.getTempDir so that we own the
// randomness (ie same seed will point to same dir):
@@ -91,7 +92,7 @@ public class TestLongPostings extends Lu
*/
}
- final OpenBitSet isS1 = new OpenBitSet(NUM_DOCS);
+ final FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
for(int idx=0;idx<NUM_DOCS;idx++) {
if (random.nextBoolean()) {
isS1.set(idx);
@@ -250,4 +251,187 @@ public class TestLongPostings extends Lu
r.close();
dir.close();
}
+
+ // a weaker form of testLongPostings, that doesnt check positions
+ public void testLongPostingsNoPositions() throws Exception {
+ doTestLongPostingsNoPositions(IndexOptions.DOCS_ONLY);
+ doTestLongPostingsNoPositions(IndexOptions.DOCS_AND_FREQS);
+ }
+
+ public void doTestLongPostingsNoPositions(IndexOptions options) throws Exception {
+ assumeFalse("Too slow with SimpleText codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
+ assumeFalse("Too slow with Memory codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
+ // Don't use _TestUtil.getTempDir so that we own the
+ // randomness (ie same seed will point to same dir):
+ Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong()));
+
+ final int NUM_DOCS = atLeast(2000);
+
+ if (VERBOSE) {
+ System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
+ }
+
+ final String s1 = getRandomTerm(null);
+ final String s2 = getRandomTerm(s1);
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: s1=" + s1 + " s2=" + s2);
+ /*
+ for(int idx=0;idx<s1.length();idx++) {
+ System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx)));
+ }
+ for(int idx=0;idx<s2.length();idx++) {
+ System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx)));
+ }
+ */
+ }
+
+ final FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
+ for(int idx=0;idx<NUM_DOCS;idx++) {
+ if (random.nextBoolean()) {
+ isS1.set(idx);
+ }
+ }
+
+ final IndexReader r;
+ if (true) {
+ final IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+ .setMergePolicy(newLogMergePolicy());
+ iwc.setRAMBufferSizeMB(16.0 + 16.0 * random.nextDouble());
+ iwc.setMaxBufferedDocs(-1);
+ final RandomIndexWriter riw = new RandomIndexWriter(random, dir, iwc);
+
+ for(int idx=0;idx<NUM_DOCS;idx++) {
+ final Document doc = new Document();
+ String s = isS1.get(idx) ? s1 : s2;
+ final Field f = newField("field", s, Field.Index.ANALYZED);
+ f.setIndexOptions(options);
+ final int count = _TestUtil.nextInt(random, 1, 4);
+ for(int ct=0;ct<count;ct++) {
+ doc.add(f);
+ }
+ riw.addDocument(doc);
+ }
+
+ r = riw.getReader();
+ riw.close();
+ } else {
+ r = IndexReader.open(dir);
+ }
+
+ /*
+ if (VERBOSE) {
+ System.out.println("TEST: terms");
+ TermEnum termEnum = r.terms();
+ while(termEnum.next()) {
+ System.out.println(" term=" + termEnum.term() + " len=" + termEnum.term().text().length());
+ assertTrue(termEnum.docFreq() > 0);
+ System.out.println(" s1?=" + (termEnum.term().text().equals(s1)) + " s1len=" + s1.length());
+ System.out.println(" s2?=" + (termEnum.term().text().equals(s2)) + " s2len=" + s2.length());
+ final String s = termEnum.term().text();
+ for(int idx=0;idx<s.length();idx++) {
+ System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx)));
+ }
+ }
+ }
+ */
+
+ assertEquals(NUM_DOCS, r.numDocs());
+ assertTrue(r.docFreq(new Term("field", s1)) > 0);
+ assertTrue(r.docFreq(new Term("field", s2)) > 0);
+
+ int num = atLeast(1000);
+ for(int iter=0;iter<num;iter++) {
+
+ final String term;
+ final boolean doS1;
+ if (random.nextBoolean()) {
+ term = s1;
+ doS1 = true;
+ } else {
+ term = s2;
+ doS1 = false;
+ }
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1);
+ }
+
+ final DocsEnum postings = MultiFields.getTermDocsEnum(r, null, "field", new BytesRef(term));
+
+ int docID = -1;
+ while(docID < DocsEnum.NO_MORE_DOCS) {
+ final int what = random.nextInt(3);
+ if (what == 0) {
+ if (VERBOSE) {
+ System.out.println("TEST: docID=" + docID + "; do next()");
+ }
+ // nextDoc
+ int expected = docID+1;
+ while(true) {
+ if (expected == NUM_DOCS) {
+ expected = Integer.MAX_VALUE;
+ break;
+ } else if (isS1.get(expected) == doS1) {
+ break;
+ } else {
+ expected++;
+ }
+ }
+ docID = postings.nextDoc();
+ if (VERBOSE) {
+ System.out.println(" got docID=" + docID);
+ }
+ assertEquals(expected, docID);
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+
+ if (random.nextInt(6) == 3) {
+ final int freq = postings.freq();
+ assertTrue(freq >=1 && freq <= 4);
+ }
+ } else {
+ // advance
+ final int targetDocID;
+ if (docID == -1) {
+ targetDocID = random.nextInt(NUM_DOCS+1);
+ } else {
+ targetDocID = docID + _TestUtil.nextInt(random, 1, NUM_DOCS - docID);
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: docID=" + docID + "; do advance(" + targetDocID + ")");
+ }
+ int expected = targetDocID;
+ while(true) {
+ if (expected == NUM_DOCS) {
+ expected = Integer.MAX_VALUE;
+ break;
+ } else if (isS1.get(expected) == doS1) {
+ break;
+ } else {
+ expected++;
+ }
+ }
+
+ docID = postings.advance(targetDocID);
+ if (VERBOSE) {
+ System.out.println(" got docID=" + docID);
+ }
+ assertEquals(expected, docID);
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+
+ if (random.nextInt(6) == 3) {
+ final int freq = postings.freq();
+ assertTrue(freq >=1 && freq <= 4);
+ }
+ }
+ }
+ }
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Mon Jul 18 13:34:29 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util._TestUtil;
@@ -26,6 +27,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -47,6 +49,7 @@ public class TestOmitTf extends LuceneTe
@Override public Explanation idfExplain(TermContext[] terms, IndexSearcher searcher) throws IOException {
return new Explanation(1.0f, "Inexplicable");
}
+ @Override public float scorePayload(int doc, int start, int end, BytesRef payload) { return 1.0f; }
};
}
}
@@ -65,7 +68,7 @@ public class TestOmitTf extends LuceneTe
// this field will NOT have Tf
Field f2 = newField("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
- f2.setOmitTermFreqAndPositions(true);
+ f2.setIndexOptions(IndexOptions.DOCS_ONLY);
d.add(f2);
writer.addDocument(d);
@@ -75,10 +78,10 @@ public class TestOmitTf extends LuceneTe
d = new Document();
// Reverse
- f1.setOmitTermFreqAndPositions(true);
+ f1.setIndexOptions(IndexOptions.DOCS_ONLY);
d.add(f1);
- f2.setOmitTermFreqAndPositions(false);
+ f2.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
d.add(f2);
writer.addDocument(d);
@@ -90,8 +93,8 @@ public class TestOmitTf extends LuceneTe
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false));
FieldInfos fi = reader.fieldInfos();
- assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions);
- assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions);
+ assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
+ assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
reader.close();
ram.close();
@@ -117,7 +120,7 @@ public class TestOmitTf extends LuceneTe
// this field will NOT have Tf
Field f2 = newField("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
- f2.setOmitTermFreqAndPositions(true);
+ f2.setIndexOptions(IndexOptions.DOCS_ONLY);
d.add(f2);
for(int i=0;i<30;i++)
@@ -128,10 +131,10 @@ public class TestOmitTf extends LuceneTe
d = new Document();
// Reverese
- f1.setOmitTermFreqAndPositions(true);
+ f1.setIndexOptions(IndexOptions.DOCS_ONLY);
d.add(f1);
- f2.setOmitTermFreqAndPositions(false);
+ f2.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
d.add(f2);
for(int i=0;i<30;i++)
@@ -144,8 +147,8 @@ public class TestOmitTf extends LuceneTe
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false));
FieldInfos fi = reader.fieldInfos();
- assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions);
- assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions);
+ assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
+ assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
reader.close();
ram.close();
@@ -176,7 +179,7 @@ public class TestOmitTf extends LuceneTe
for(int i=0;i<5;i++)
writer.addDocument(d);
- f2.setOmitTermFreqAndPositions(true);
+ f2.setIndexOptions(IndexOptions.DOCS_ONLY);
for(int i=0;i<20;i++)
writer.addDocument(d);
@@ -189,8 +192,8 @@ public class TestOmitTf extends LuceneTe
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false));
FieldInfos fi = reader.fieldInfos();
- assertTrue("OmitTermFreqAndPositions field bit should not be set.", !fi.fieldInfo("f1").omitTermFreqAndPositions);
- assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions);
+ assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
+ assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
reader.close();
ram.close();
@@ -198,8 +201,10 @@ public class TestOmitTf extends LuceneTe
private void assertNoPrx(Directory dir) throws Throwable {
final String[] files = dir.listAll();
- for(int i=0;i<files.length;i++)
+ for(int i=0;i<files.length;i++) {
assertFalse(files[i].endsWith(".prx"));
+ assertFalse(files[i].endsWith(".pos"));
+ }
}
// Verifies no *.prx exists when all fields omit term freq:
@@ -213,8 +218,8 @@ public class TestOmitTf extends LuceneTe
lmp.setUseCompoundFile(false);
Document d = new Document();
- Field f1 = newField("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
- f1.setOmitTermFreqAndPositions(true);
+ Field f1 = newField("f1", "This field has no term freqs", Field.Store.NO, Field.Index.ANALYZED);
+ f1.setIndexOptions(IndexOptions.DOCS_ONLY);
d.add(f1);
for(int i=0;i<30;i++)
@@ -223,7 +228,15 @@ public class TestOmitTf extends LuceneTe
writer.commit();
assertNoPrx(ram);
-
+
+ // now add some documents with positions, and check there is no prox after optimization
+ d = new Document();
+ f1 = newField("f1", "This field has positions", Field.Store.NO, Field.Index.ANALYZED);
+ d.add(f1);
+
+ for(int i=0;i<30;i++)
+ writer.addDocument(d);
+
// force merge
writer.optimize();
// flush
@@ -253,7 +266,7 @@ public class TestOmitTf extends LuceneTe
sb.append(term).append(" ");
String content = sb.toString();
Field noTf = newField("noTf", content + (i%2==0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED);
- noTf.setOmitTermFreqAndPositions(true);
+ noTf.setIndexOptions(IndexOptions.DOCS_ONLY);
d.add(noTf);
Field tf = newField("tf", content + (i%2==0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java Mon Jul 18 13:34:29 2011
@@ -47,6 +47,22 @@ public class TestSameTokenSamePosition e
riw.close();
dir.close();
}
+
+ /**
+ * Same as the above, but with more docs
+ */
+ public void testMoreDocs() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter riw = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new BugReproAnalyzer()));
+ Document doc = new Document();
+ doc.add(new Field("eng", "Six drunken" /*This shouldn't matter. */,
+ Field.Store.YES, Field.Index.ANALYZED));
+ for (int i = 0; i < 100; i++) {
+ riw.addDocument(doc);
+ }
+ riw.close();
+ dir.close();
+ }
}
final class BugReproAnalyzer extends Analyzer{
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Mon Jul 18 13:34:29 2011
@@ -49,7 +49,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FloatsRef;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util._TestUtil;
import org.junit.Before;
@@ -233,11 +233,11 @@ public class TestDocValuesIndexing exten
// run in random order to test if fill works correctly during merges
Collections.shuffle(numVariantList, random);
for (ValueType val : numVariantList) {
- OpenBitSet deleted = indexValues(w, numValues, val, numVariantList,
+ FixedBitSet deleted = indexValues(w, numValues, val, numVariantList,
withDeletions, 7);
List<Closeable> closeables = new ArrayList<Closeable>();
IndexReader r = IndexReader.open(w, true);
- final int numRemainingValues = (int) (numValues - deleted.cardinality());
+ final int numRemainingValues = numValues - deleted.cardinality();
final int base = r.numDocs() - numRemainingValues;
// for FIXED_INTS_8 we use value mod 128 - to enable testing in
// one go we simply use numValues as the mod for all other INT types
@@ -331,11 +331,11 @@ public class TestDocValuesIndexing exten
for (ValueType byteIndexValue : byteVariantList) {
List<Closeable> closeables = new ArrayList<Closeable>();
final int bytesSize = 1 + atLeast(50);
- OpenBitSet deleted = indexValues(w, numValues, byteIndexValue,
+ FixedBitSet deleted = indexValues(w, numValues, byteIndexValue,
byteVariantList, withDeletions, bytesSize);
final IndexReader r = IndexReader.open(w, withDeletions);
assertEquals(0, r.numDeletedDocs());
- final int numRemainingValues = (int) (numValues - deleted.cardinality());
+ final int numRemainingValues = numValues - deleted.cardinality();
final int base = r.numDocs() - numRemainingValues;
IndexDocValues bytesReader = getDocValues(r, byteIndexValue.name());
assertNotNull("field " + byteIndexValue.name()
@@ -484,11 +484,11 @@ public class TestDocValuesIndexing exten
Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS,
Index.NO };
- private OpenBitSet indexValues(IndexWriter w, int numValues, ValueType value,
+ private FixedBitSet indexValues(IndexWriter w, int numValues, ValueType value,
List<ValueType> valueVarList, boolean withDeletions, int bytesSize)
throws CorruptIndexException, IOException {
final boolean isNumeric = NUMERICS.contains(value);
- OpenBitSet deleted = new OpenBitSet(numValues);
+ FixedBitSet deleted = new FixedBitSet(numValues);
Document doc = new Document();
Index idx = IDX_VALUES[random.nextInt(IDX_VALUES.length)];
AbstractField field = random.nextBoolean() ? new IndexDocValuesField(value.name())
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java Mon Jul 18 13:34:29 2011
@@ -67,7 +67,7 @@ public class BaseTestRangeFilter extends
static TestIndex unsignedIndexDir;
static int minId = 0;
- static int maxId = atLeast(500);
+ static int maxId;
static final int intLength = Integer.toString(Integer.MAX_VALUE).length();
@@ -93,6 +93,7 @@ public class BaseTestRangeFilter extends
@BeforeClass
public static void beforeClassBaseTestRangeFilter() throws Exception {
+ maxId = atLeast(500);
signedIndexDir = new TestIndex(random, Integer.MAX_VALUE, Integer.MIN_VALUE, true);
unsignedIndexDir = new TestIndex(random, Integer.MAX_VALUE, 0, false);
signedIndexReader = build(random, signedIndexDir);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Mon Jul 18 13:34:29 2011
@@ -265,11 +265,6 @@ final class JustCompileSearch {
public byte computeNorm(FieldInvertState state) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
-
- @Override
- public float sloppyFreq(int distance) {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
}
static final class JustCompileSimilarityProvider implements SimilarityProvider {
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Mon Jul 18 13:34:29 2011
@@ -30,8 +30,7 @@ import org.apache.lucene.index.SlowMulti
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
-import org.apache.lucene.util.OpenBitSetDISI;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util._TestUtil;
public class TestCachingWrapperFilter extends LuceneTestCase {
@@ -125,7 +124,7 @@ public class TestCachingWrapperFilter ex
if (originalSet.isCacheable()) {
assertEquals("Cached DocIdSet must be of same class like uncached, if cacheable", originalSet.getClass(), cachedSet.getClass());
} else {
- assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI || cachedSet == DocIdSet.EMPTY_DOCIDSET);
+ assertTrue("Cached DocIdSet must be an FixedBitSet if the original one was not cacheable", cachedSet instanceof FixedBitSet || cachedSet == DocIdSet.EMPTY_DOCIDSET);
}
}
@@ -143,11 +142,11 @@ public class TestCachingWrapperFilter ex
assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true);
// is cacheable:
assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true);
- // a openbitset filter is always cacheable
+ // a fixedbitset filter is always cacheable
assertDocIdSetCacheable(reader, new Filter() {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context) {
- return new OpenBitSet();
+ return new FixedBitSet(context.reader.maxDoc());
}
}, true);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestCustomSearcherSort.java Mon Jul 18 13:34:29 2011
@@ -39,7 +39,7 @@ public class TestCustomSearcherSort exte
private IndexReader reader;
private Query query = null;
// reduced from 20000 to 2000 to speed up test...
- private final static int INDEX_SIZE = atLeast(2000);
+ private int INDEX_SIZE;
/**
* Create index and query for test cases.
@@ -47,6 +47,7 @@ public class TestCustomSearcherSort exte
@Override
public void setUp() throws Exception {
super.setUp();
+ INDEX_SIZE = atLeast(2000);
index = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, index);
RandomGen random = new RandomGen(this.random);
Modified: lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1147873&r1=1147872&r2=1147873&view=diff
==============================================================================
--- lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/branches/positions/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java Mon Jul 18 13:34:29 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TermContext;
@@ -145,11 +146,6 @@ public class TestDocValuesScoring extend
}
@Override
- public float sloppyFreq(int distance) {
- return sim.sloppyFreq(distance);
- }
-
- @Override
public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
return sim.computeStats(searcher, fieldName, queryBoost, termContexts);
}
@@ -189,6 +185,16 @@ public class TestDocValuesScoring extend
}
@Override
+ public float computeSlopFactor(int distance) {
+ return sub.computeSlopFactor(distance);
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return sub.computePayloadFactor(doc, start, end, payload);
+ }
+
+ @Override
public Explanation explain(int doc, Explanation freq) {
Explanation boostExplanation = new Explanation((float) values.getFloat(doc), "indexDocValue(" + boostField + ")");
Explanation simExplanation = sub.explain(doc, freq);