You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/03/28 12:50:48 UTC
svn commit: r1086181 [6/20] - in /lucene/dev/branches/docvalues: ./
dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-to...
Modified: lucene/dev/branches/docvalues/lucene/docs/gettingstarted.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/gettingstarted.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/gettingstarted.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/gettingstarted.html Mon Mar 28 10:50:28 2011
@@ -129,8 +129,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', 'skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', 'skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="api/contrib-ant/index.html">Ant</a>
</div>
Modified: lucene/dev/branches/docvalues/lucene/docs/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/index.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/index.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/index.html Mon Mar 28 10:50:28 2011
@@ -127,8 +127,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', 'skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', 'skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="api/contrib-ant/index.html">Ant</a>
</div>
Modified: lucene/dev/branches/docvalues/lucene/docs/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/linkmap.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/linkmap.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/linkmap.html Mon Mar 28 10:50:28 2011
@@ -127,8 +127,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', 'skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', 'skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="api/contrib-ant/index.html">Ant</a>
</div>
@@ -298,6 +301,12 @@ document.write("Last Published: " + docu
<ul>
<li>
+<a href="api/test-framework/index.html">Test Framework</a> ___________________ <em>javadoc-test-framework</em>
+</li>
+</ul>
+
+<ul>
+<li>
<a>Contrib</a> ___________________ <em>javadoc-contrib</em>
</li>
<ul>
Modified: lucene/dev/branches/docvalues/lucene/docs/linkmap.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/linkmap.pdf?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/linkmap.pdf (original)
+++ lucene/dev/branches/docvalues/lucene/docs/linkmap.pdf Mon Mar 28 10:50:28 2011
@@ -5,10 +5,10 @@
/Producer (FOP 0.20.5) >>
endobj
5 0 obj
-<< /Length 1080 /Filter [ /ASCII85Decode /FlateDecode ]
+<< /Length 1086 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
-Gatn&>uTc+(kqGU.rqU*AUshQ0!2:_lfd[rp28^>_3EYe5&-83l+^RrdKWT(%]X"JA9uN'Gn5JZI.H[QpFlWpRkH-crJN&MH^l2hb$X3uWfI_tHkq&"bklI%Qdk$ES)PQTG>><)r&Ng'^15G1;XJq<m'"`n]G>cuSU,e!q3?'!*l>%c@7PqM0M#BZq'rA."BBY@oP<;jFp)ZW[;<&4Q090,Of_PZCHY[l[s5X6S8-<KWln+ADb:$?m)[%=,7oPLq`VskS\ZMKVM-c2aF:MJRon_YM8I:-hso#YO<"X:(',L3;mIF=*<J##5nLiY^n)b"%fW2Y2jo&2]M=@>(!S*iLa$XR/W0>:@R*V0:b4g_BYV!Mga;%2R>Uk.K]sPs6$/%o@Lu_ac<!,X'UAQ;"i/0PBbBekm9GdR-kcB)IYM)rl!.OYUCEH]ha'us7X0lNOhNN1aRZo9*h;3M)T)Vm>4cd)p;dmnKN_l)`OEK3=P$%m_hh*GVcIkdhGd#t>\@d9?N.7QHtLNF9N/*9a1*7u-f?L):"S9TIAkTl0R+rI?W(Tm71V=XC"QL*>PU=f-K-MZS_52F'H`E2\PaNEPI5/K_//'oF6_.D=[RYZpTXG.m.o"s#K\oQEiqNV+CL.q%W<IJ(1WS,bEe0VaQ$:?b@bHahcJ!XiGlfhRuF$"#[-2.+gStQ'"76`6[sp&BFMld")JFjO5t;Gb.h,ET:/M8T8(T%FJ\knn]Itf/OD\nWg5A5GV']'!EL-h_e07F(e/#RlI/'NpSDT`?3K*>l39W*Zum3q7)U"C)d,Ylc9:r">H,3O&2NQ&7OV,%X_G@<W`Z&GcUG*YhFirBHUpi*,?X1eNZ'TeXDTtk9\fBH"K@`tkqHXqfisDDb+4DA9aYsnT@JD^Z9%(<TL+np,$G"RQ95ha`MZdF6hIMNmFF>Gp\EmH3FOke73pL*J9n"WLB(8V']*kN4.3(3\J0gl4.1j`+B1Fi,>(8?W#,8
[#-I]HZ/1a\H0UIGpR%\hKct!\8U[G+e'HT-f%YDfZ@:o3eoOu1njRU("PVIrc"GXA]IU*9jKQ+&!W&Mi#OPXn^]~>
+Gau1.b>R(K'Sc@-$85g"<Hm1j/nJ2jVX9INl#_TR&;+JFXhaH"Q#$[u:5'P6_@S7Y3WH#;o.*>OMqP!a&n),TZrkSGq3T?6F?;i%eA!D(boEVm&=u:J:%\&8.Tc0pRc?BTG7H7rnpVf)L5CVo.BNEsC:HI,8gR-\*(I];`OT#]&]1*E,KJd\E(E($`B%p2#\NKEf[jTBlj:6h-eVe+[2]BR[<X0rZ=]Q\*f4oO.4Ioi;=uhGgKK21G3!(E.7fYor>K2>`aTN:Bjht_\m+R@2Nst'@^$UH^&R[BORE@k49Yk-2e9Be56U*QJg,`=K0;`'>Pr3#D)_=%$'eTR<&?oJ85nj^/Z\6?@R/^B0W\VEB]#ClrKPLga]+4@$*uLhf:X9BLdC#D4'@L4<]8FU'a'%$=!HD<oQ-k+U1cjts,=LgH?+9Cc<;VDILe<^g;"je:5?^A,:ccbhb8dRHOPt?.l"j4Ylddd;WbC'\Te`m_+oESg76f\\EdUYo4C\]iArHGl/f+LN,U_'cSQFbDMS[@^B=I(V:kB_@.1*PP/LZ;/7bN1^&>#J)u3ahpJYKLL@(;q8FBlNes`cHM%DR@rjP05M'8c?K4&/W"B36&K,g.k10_oA_+/$W@A8P@#kZj3GW@p.k0`Cs:A?R$c12n.6dTTbbr<D,C`'/sF&$"p^EBrt!D8k+YB>k:RB8F-"(i^pm"18;*p4XY6B/8%/uN_g9r<U"f<5j;N4Ea.6A2qfn/:@]I_.6)T9Al&dV88dbD\a=[SDY1]R"h;QAs<Nb_;5'NrYY5d*W^5kDiAp(f@K9F?<dhdu0mFXXu4r/NSRc,>+BU;O=tRlBgbT:YSKa'ZQ6(?6VcTF0D&co-TYi+\GmS-au@Jm:rUNZj3Xs_Pk%en_*3J<n@*=esdl4HajDm`lFW<#a6N?2]9i+TagfMlK6WhT,4iSp))YX.?WJH@M[AbN=]J5eM:=QIa!hsnQhnVcl_-Y*)gI?h@X6
?jXa"1?u\\V;E_\BAht:Ljk#ln@>#fbeo?.U*TiD(6tVL`Wg.lG[5E#+D9HL;:^#@ig((M4[+^KsQ?q4[EX@JsrrF8>#*A~>
endstream
endobj
6 0 obj
@@ -20,10 +20,10 @@ endobj
>>
endobj
7 0 obj
-<< /Length 886 /Filter [ /ASCII85Decode /FlateDecode ]
+<< /Length 901 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
-GatUr?#SFN'Sc)P'u"K;nGEm=c=L4>8Q."="f[^:Bl\hnZ\ASq;p)kIo0m,M)_&SQ1&-/=+.m'?FtreJ@fR0u^`o3iYaSsMbSL@J4Yu:X+8X%\M`(9Xoh5Y'##gHu1-]&-Wcb2().9D#s.4_C_&ooVDERil8#u)e"t:k,_a!rLQWZ9WZW-,Xb;=,g)ARFmdSNgO4U:(r,BYQ$nX.s:qR4?lB$ko5EFJs*P>`c!po;1iQ&gZr!=XWuF3`M6)$"87!0IY6+/c0eUe!bC'AMFR[<<Brj8fr)`ht@pVFlp,r%an][Q(,=[iRo,5'"*[Y93Kc,'ECZ;u*,L4%c_TPBjhcr$h6sJl4nI?CLU[?%8r>l*dCUDAk=o)R3WMP"uG_:h[9)QMIJf]@\sM8p,,FGFl;[Ptc.Q%oFQqQ/C,kf]b@g7q5G7"u*<A6<e_cd1bptTA;Vg-DTd(4HeH-)Xaci=e)F.QY0I8QK'okqV5UTfS5N.Qq"4I&1YTC]A]7-\j"%Ng:<N[<m1%1/79%a[\r!H/H>WkBLjRl2UD=S<,""f9j*J\'r^DHb;-%Zg?'Ks`RNQubn5+mlQOgLj-&i(`3Tp=A#X-p#>f,"S]rr$c0gP*aJ_V0B8gD![$j?egiKpO;u@D<f%$)_^638D&0RWYSYLj'5q(Y<`L$f$P^-NImKba1"o!j&_DlsJbDeBfO=uIhY:QurZ;Q%InCPWi>J2hF&eUU;LbmL5AUkn3+BK\S4A9Y9LFGF+A-S)%$3C_F*jjPWb4[3rARa.OCsD7Ta*Q'9bSUb9#)$V)N,,E)Cid\XeZ5kF7%*d^_Jg;'K`!H+]W*TofuUB>ca-12A_nAoA\3%5bL7O+VBneM<8L%s<p%\U=%h3-mQYIufZs~>
+GatUr>>O!-(l%MV.sqX>omDR*k7Z-&Bk#M`OdIu&aeV\R8<Yr...@I>FFS&]*<b8s4rL9PLrH_3e\.,ctS0)$";FnN\No`:KNTlPA4YE(b4R\k<C>BpIsJ\$rT]6LH]HR>DuEg!Qb?'E[u8Xj6;r`@(Nd8+#@gt7Q?h!5r?Z^i])0__-<%fM6PK^T!3(W=h?k-^taQO#l"rWlcGqn5aN/=i!d#6(h7Yb>O+GlV_hU(8l&63ELt%UkYiN@O"jB7&%9oi+87*+<U`iWB.[c*2'\Y9r)n75e-,8UCZIffIr++Z>0KLW"a3O(FJ6L2\Q52CRu_<nO%b2,7-C3,cCSfXg16NE#dkn3X8K+]MS5%uA25LnAcWS".'SuW)#8-905_gXP>TgG&*7]!L(2c*,)TBij[GVTjR`RRQ^cJ3gFtGoo<8fJ6`">_DYU*9Y(g%Q]@8UbC,J0#!d^GRHg8:>ZcfgF9oA1CoA&d:Q#r%o0$DA0!gVS[=iR:.e,5uNe>%Y%;ie03YoXQ15<dOZPW6*[24!f:GRTj!K?^_29Qm:?ACR=71WH`A)pL)i`Wk2].=-n9f@?2`Tp2Um+@&34R\PO$5q(Zg*\d,a:R4lR\g-"c'\k"3#c$Ht,Ic6m04/2c^XCI9Mb<A1iNN5s=$Ba"!YLo+LbmN97=ZLh+B'DQ4j7f%(G>,c14r(#"eGf"pR5l:D>TOB2DM4=aC]YO@@_Tt2M^>'$jami(fsUZl1UK5lW7BWM;m<_$,^UJ&9?OHIoU?)Z^!g8?DaD[aK2_R8GC4u(=O=J5)\GZ.)#4IXN[74Y#_bi]DlJMm)/~>
endstream
endobj
8 0 obj
@@ -87,19 +87,19 @@ endobj
xref
0 14
0000000000 65535 f
-0000002989 00000 n
-0000003053 00000 n
-0000003103 00000 n
+0000003010 00000 n
+0000003074 00000 n
+0000003124 00000 n
0000000015 00000 n
0000000071 00000 n
-0000001243 00000 n
-0000001349 00000 n
-0000002326 00000 n
-0000002432 00000 n
-0000002544 00000 n
-0000002654 00000 n
-0000002765 00000 n
-0000002873 00000 n
+0000001249 00000 n
+0000001355 00000 n
+0000002347 00000 n
+0000002453 00000 n
+0000002565 00000 n
+0000002675 00000 n
+0000002786 00000 n
+0000002894 00000 n
trailer
<<
/Size 14
@@ -107,5 +107,5 @@ trailer
/Info 4 0 R
>>
startxref
-3225
+3246
%%EOF
Modified: lucene/dev/branches/docvalues/lucene/docs/lucene-contrib/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/lucene-contrib/index.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/lucene-contrib/index.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/lucene-contrib/index.html Mon Mar 28 10:50:28 2011
@@ -129,8 +129,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="../api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', '../skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="../api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', '../skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="../api/contrib-ant/index.html">Ant</a>
</div>
Modified: lucene/dev/branches/docvalues/lucene/docs/queryparsersyntax.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/queryparsersyntax.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/queryparsersyntax.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/queryparsersyntax.html Mon Mar 28 10:50:28 2011
@@ -129,8 +129,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', 'skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', 'skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="api/contrib-ant/index.html">Ant</a>
</div>
Modified: lucene/dev/branches/docvalues/lucene/docs/scoring.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/scoring.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/scoring.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/scoring.html Mon Mar 28 10:50:28 2011
@@ -129,8 +129,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', 'skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', 'skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="api/contrib-ant/index.html">Ant</a>
</div>
Modified: lucene/dev/branches/docvalues/lucene/docs/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/docs/systemrequirements.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/docs/systemrequirements.html (original)
+++ lucene/dev/branches/docvalues/lucene/docs/systemrequirements.html Mon Mar 28 10:50:28 2011
@@ -127,8 +127,11 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/core/index.html">Core</a>
</div>
-<div onclick="SwitchMenu('menu_1.1.3.3', 'skin/')" id="menu_1.1.3.3Title" class="menutitle">Contrib</div>
-<div id="menu_1.1.3.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework</a>
+</div>
+<div onclick="SwitchMenu('menu_1.1.3.4', 'skin/')" id="menu_1.1.3.4Title" class="menutitle">Contrib</div>
+<div id="menu_1.1.3.4" class="menuitemgroup">
<div class="menuitem">
<a href="api/contrib-ant/index.html">Ant</a>
</div>
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java Mon Mar 28 10:50:28 2011
@@ -142,8 +142,13 @@ public final class NumericTokenStream ex
public static final class NumericTermAttributeImpl extends AttributeImpl implements NumericTermAttribute,TermToBytesRefAttribute {
private long value = 0L;
private int valueSize = 0, shift = 0, precisionStep = 0;
+ private BytesRef bytes = new BytesRef();
+
+ public BytesRef getBytesRef() {
+ return bytes;
+ }
- public int toBytesRef(BytesRef bytes) {
+ public int fillBytesRef() {
try {
assert valueSize == 64 || valueSize == 32;
return (valueSize == 64) ?
@@ -180,9 +185,8 @@ public final class NumericTokenStream ex
@Override
public void reflectWith(AttributeReflector reflector) {
- final BytesRef bytes = new BytesRef();
- toBytesRef(bytes);
- reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes);
+ fillBytesRef();
+ reflector.reflect(TermToBytesRefAttribute.class, "bytes", new BytesRef(bytes));
reflector.reflect(NumericTermAttribute.class, "shift", shift);
reflector.reflect(NumericTermAttribute.class, "rawValue", getRawValue());
reflector.reflect(NumericTermAttribute.class, "valueSize", valueSize);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/package.html?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/package.html (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/package.html Mon Mar 28 10:50:28 2011
@@ -130,7 +130,7 @@ There are many post tokenization steps t
</li>
</ul>
However an application might invoke Analysis of any text for testing or for any other purpose, something like:
- <PRE>
+ <PRE class="prettyprint">
Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
while (ts.incrementToken()) {
@@ -182,7 +182,7 @@ the source code of any one of the many s
This allows phrase search and proximity search to seamlessly cross
boundaries between these "sections".
In other words, if a certain field "f" is added like this:
- <PRE>
+ <PRE class="prettyprint">
document.add(new Field("f","first ends",...);
document.add(new Field("f","starts two",...);
indexWriter.addDocument(document);
@@ -191,7 +191,7 @@ the source code of any one of the many s
Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections",
simply by overriding
{@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
- <PRE>
+ <PRE class="prettyprint">
Analyzer myAnalyzer = new StandardAnalyzer() {
public int getPositionIncrementGap(String fieldName) {
return 10;
@@ -220,7 +220,7 @@ the source code of any one of the many s
tokens following a removed stop word, using
{@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#setPositionIncrement(int)}.
This can be done with something like:
- <PRE>
+ <PRE class="prettyprint">
public TokenStream tokenStream(final String fieldName, Reader reader) {
final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
TokenStream res = new TokenStream() {
@@ -334,7 +334,7 @@ here to illustrate the usage of the new
Then we will develop a custom Attribute, a PartOfSpeechAttribute, and add another filter to the chain which
utilizes the new custom attribute, and call it PartOfSpeechTaggingFilter.
<h4>Whitespace tokenization</h4>
-<pre>
+<pre class="prettyprint">
public class MyAnalyzer extends Analyzer {
public TokenStream tokenStream(String fieldName, Reader reader) {
@@ -381,7 +381,7 @@ API
<h4>Adding a LengthFilter</h4>
We want to suppress all tokens that have 2 or less characters. We can do that easily by adding a LengthFilter
to the chain. Only the tokenStream() method in our analyzer needs to be changed:
-<pre>
+<pre class="prettyprint">
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream stream = new WhitespaceTokenizer(reader);
stream = new LengthFilter(stream, 3, Integer.MAX_VALUE);
@@ -398,7 +398,7 @@ TokenStream
API
</pre>
Now let's take a look how the LengthFilter is implemented (it is part of Lucene's core):
-<pre>
+<pre class="prettyprint">
public final class LengthFilter extends TokenFilter {
final int min;
@@ -448,7 +448,7 @@ is neccessary. The same is true for the
<h4>Adding a custom Attribute</h4>
Now we're going to implement our own custom Attribute for part-of-speech tagging and call it consequently
<code>PartOfSpeechAttribute</code>. First we need to define the interface of the new Attribute:
-<pre>
+<pre class="prettyprint">
public interface PartOfSpeechAttribute extends Attribute {
public static enum PartOfSpeech {
Noun, Verb, Adjective, Adverb, Pronoun, Preposition, Conjunction, Article, Unknown
@@ -470,7 +470,7 @@ and returns an actual instance. You can
Now here is the actual class that implements our new Attribute. Notice that the class has to extend
{@link org.apache.lucene.util.AttributeImpl}:
-<pre>
+<pre class="prettyprint">
public final class PartOfSpeechAttributeImpl extends AttributeImpl
implements PartOfSpeechAttribute{
@@ -513,7 +513,7 @@ This is a simple Attribute implementatio
new <code>AttributeImpl</code> class and therefore implements its abstract methods <code>clear(), copyTo(), equals(), hashCode()</code>.
Now we need a TokenFilter that can set this new PartOfSpeechAttribute for each token. In this example we show a very naive filter
that tags every word with a leading upper-case letter as a 'Noun' and all other words as 'Unknown'.
-<pre>
+<pre class="prettyprint">
public static class PartOfSpeechTaggingFilter extends TokenFilter {
PartOfSpeechAttribute posAtt;
CharTermAttribute termAtt;
@@ -544,7 +544,7 @@ Just like the LengthFilter, this new fil
stores references in instance variables. Notice how you only need to pass in the interface of the new
Attribute and instantiating the correct class is automatically been taken care of.
Now we need to add the filter to the chain:
-<pre>
+<pre class="prettyprint">
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream stream = new WhitespaceTokenizer(reader);
stream = new LengthFilter(stream, 3, Integer.MAX_VALUE);
@@ -564,7 +564,7 @@ API
Apparently it hasn't changed, which shows that adding a custom attribute to a TokenStream/Filter chain does not
affect any existing consumers, simply because they don't know the new Attribute. Now let's change the consumer
to make use of the new PartOfSpeechAttribute and print it out:
-<pre>
+<pre class="prettyprint">
public static void main(String[] args) throws IOException {
// text to tokenize
final String text = "This is a demo of the new TokenStream API";
@@ -606,7 +606,7 @@ API the reader could now write an Attrib
of a sentence or not. Then the PartOfSpeechTaggingFilter can make use of this knowledge and only tag capitalized words
as nouns if not the first word of a sentence (we know, this is still not a correct behavior, but hey, it's a good exercise).
As a small hint, this is how the new Attribute class could begin:
-<pre>
+<pre class="prettyprint">
public class FirstTokenOfSentenceAttributeImpl extends Attribute
implements FirstTokenOfSentenceAttribute {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java Mon Mar 28 10:50:28 2011
@@ -77,8 +77,16 @@ public class CharTermAttributeImpl exten
}
// *** TermToBytesRefAttribute interface ***
- public final int toBytesRef(BytesRef target) {
- return UnicodeUtil.UTF16toUTF8WithHash(termBuffer, 0, termLength, target);
+ private BytesRef bytes = new BytesRef(MIN_BUFFER_SIZE);
+
+ // not until java 6 @Override
+ public int fillBytesRef() {
+ return UnicodeUtil.UTF16toUTF8WithHash(termBuffer, 0, termLength, bytes);
+ }
+
+ // not until java 6 @Override
+ public BytesRef getBytesRef() {
+ return bytes;
}
// *** CharSequence interface ***
@@ -205,6 +213,7 @@ public class CharTermAttributeImpl exten
// Do a deep clone
t.termBuffer = new char[this.termLength];
System.arraycopy(this.termBuffer, 0, t.termBuffer, 0, this.termLength);
+ t.bytes = new BytesRef(bytes);
return t;
}
@@ -246,9 +255,8 @@ public class CharTermAttributeImpl exten
@Override
public void reflectWith(AttributeReflector reflector) {
reflector.reflect(CharTermAttribute.class, "term", toString());
- final BytesRef bytes = new BytesRef();
- toBytesRef(bytes);
- reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes);
+ fillBytesRef();
+ reflector.reflect(TermToBytesRefAttribute.class, "bytes", new BytesRef(bytes));
}
@Override
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java Mon Mar 28 10:50:28 2011
@@ -22,18 +22,39 @@ import org.apache.lucene.util.BytesRef;
/**
* This attribute is requested by TermsHashPerField to index the contents.
- * This attribute has no real state, it should be implemented in addition to
- * {@link CharTermAttribute}, to support indexing the term text as
- * UTF-8 bytes.
+ * This attribute can be used to customize the final byte[] encoding of terms.
+ * <p>
+ * Consumers of this attribute call {@link #getBytesRef()} up-front, and then
+ * invoke {@link #fillBytesRef()} for each term. Example:
+ * <pre class="prettyprint">
+ * final TermToBytesRefAttribute termAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
+ * final BytesRef bytes = termAtt.getBytesRef();
+ *
+ * while (termAtt.incrementToken() {
+ *
+ * // you must call termAtt.fillBytesRef() before doing something with the bytes.
+ * // this encodes the term value (internally it might be a char[], etc) into the bytes.
+ * int hashCode = termAtt.fillBytesRef();
+ *
+ * if (isInteresting(bytes)) {
+ *
+ * // because the bytes are reused by the attribute (like CharTermAttribute's char[] buffer),
+ * // you should make a copy if you need persistent access to the bytes, otherwise they will
+ * // be rewritten across calls to incrementToken()
+ *
+ * doSomethingWith(new BytesRef(bytes));
+ * }
+ * }
+ * ...
+ * </pre>
* @lucene.experimental This is a very expert API, please use
* {@link CharTermAttributeImpl} and its implementation of this method
* for UTF-8 terms.
*/
public interface TermToBytesRefAttribute extends Attribute {
- /** Copies the token's term text into the given {@link BytesRef}.
- * @param termBytes destination to write the bytes to (UTF-8 for text terms).
- * The length of the BytesRef's buffer may be not large enough, so you need to grow.
- * The parameters' {@code bytes} is guaranteed to be not {@code null}.
+ /**
+ * Updates the bytes {@link #getBytesRef()} to contain this term's
+ * final encoding, and returns its hashcode.
* @return the hashcode as defined by {@link BytesRef#hashCode}:
* <pre>
* int hash = 0;
@@ -45,5 +66,12 @@ public interface TermToBytesRefAttribute
* the hash on-the-fly. If this is not the case, just return
* {@code termBytes.hashCode()}.
*/
- public int toBytesRef(BytesRef termBytes);
+ public int fillBytesRef();
+
+ /**
+ * Retrieve this attribute's BytesRef. The bytes are updated
+ * from the current term when the consumer calls {@link #fillBytesRef()}.
+ * @return this Attributes internal BytesRef.
+ */
+ public BytesRef getBytesRef();
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java Mon Mar 28 10:50:28 2011
@@ -348,8 +348,14 @@ public class CheckIndex {
if (format == DefaultSegmentInfosWriter.FORMAT_DIAGNOSTICS) {
sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
+ } else if (format == DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
+ sFormat = "FORMAT_HAS_VECTORS [Lucene 3.1]";
+ } else if (format == DefaultSegmentInfosWriter.FORMAT_3_1) {
+ sFormat = "FORMAT_3_1 [Lucene 3.1]";
} else if (format == DefaultSegmentInfosWriter.FORMAT_4_0) {
sFormat = "FORMAT_4_0 [Lucene 4.0]";
+ } else if (format == DefaultSegmentInfosWriter.FORMAT_CURRENT) {
+ throw new RuntimeException("BUG: You should update this tool!");
} else if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) {
sFormat = "int=" + format + " [newer version of Lucene than this tool supports]";
skip = true;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java Mon Mar 28 10:50:28 2011
@@ -27,7 +27,8 @@ abstract class DocConsumerPerThread {
* DocumentsWriter.DocWriter and return it.
* DocumentsWriter then calls finish() on this object
* when it's its turn. */
- abstract DocumentsWriter.DocWriter processDocument() throws IOException;
+ abstract DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos) throws IOException;
+ abstract void doAfterFlush();
abstract void abort();
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java Mon Mar 28 10:50:28 2011
@@ -22,9 +22,6 @@ import java.util.Collection;
import java.util.Map;
abstract class DocFieldConsumer {
-
- FieldInfos fieldInfos;
-
/** Called when DocumentsWriter decides to create a new
* segment */
abstract void flush(Map<DocFieldConsumerPerThread,Collection<DocFieldConsumerPerField>> threadsAndFields, SegmentWriteState state) throws IOException;
@@ -39,8 +36,4 @@ abstract class DocFieldConsumer {
* The consumer should free RAM, if possible, returning
* true if any RAM was in fact freed. */
abstract boolean freeRAM();
-
- void setFieldInfos(FieldInfos fieldInfos) {
- this.fieldInfos = fieldInfos;
}
-}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Mon Mar 28 10:50:28 2011
@@ -38,7 +38,6 @@ import org.apache.lucene.store.Directory
final class DocFieldProcessor extends DocConsumer {
final DocumentsWriter docWriter;
- final FieldInfos fieldInfos;
final DocFieldConsumer consumer;
final StoredFieldsWriter fieldsWriter;
final private Map<String, DocValuesConsumer> docValues = new HashMap<String, DocValuesConsumer>();
@@ -59,7 +58,7 @@ final class DocFieldProcessor extends Do
* the SegmentsWriteState passed in right at the moment when the segment is flushed (doccount etc) but we need the consumer earlier
* to support docvalues and later on stored fields too.
*/
- docValuesConsumerState = docWriter.segWriteState();
+ docValuesConsumerState = docWriter.segWriteState(false);
fieldsConsumer = docValuesConsumerState.segmentCodecs.codec().fieldsConsumer(docValuesConsumerState);
}
valuesConsumer = fieldsConsumer.addValuesField(fieldInfo);
@@ -73,9 +72,7 @@ final class DocFieldProcessor extends Do
public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer) {
this.docWriter = docWriter;
this.consumer = consumer;
- fieldInfos = docWriter.getFieldInfos();
- consumer.setFieldInfos(fieldInfos);
- fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos);
+ fieldsWriter = new StoredFieldsWriter(docWriter);
}
@Override
@@ -85,7 +82,6 @@ final class DocFieldProcessor extends Do
for ( DocConsumerPerThread thread : threads) {
DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread) thread;
childThreadsAndFields.put(perThread.consumer, perThread.fields());
- perThread.trimFields(state);
}
fieldsWriter.flush(state);
consumer.flush(childThreadsAndFields, state);
@@ -93,13 +89,11 @@ final class DocFieldProcessor extends Do
for(DocValuesConsumer p : docValues.values()) {
if (p != null) {
p.finish(state.numDocs);
- p.files(state.flushedFiles);
}
}
docValues.clear();
if(fieldsConsumer != null) {
fieldsConsumer.close(); // TODO remove this once docvalues are fully supported by codecs
- state.flushedFiles.addAll(docValuesConsumerState.flushedFiles);
docValuesConsumerState = null;
fieldsConsumer = null;
}
@@ -109,7 +103,7 @@ final class DocFieldProcessor extends Do
// FreqProxTermsWriter does this with
// FieldInfo.storePayload.
final String fileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELD_INFOS_EXTENSION);
- fieldInfos.write(state.directory, fileName);
+ state.fieldInfos.write(state.directory, fileName);
}
@Override
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Mon Mar 28 10:50:28 2011
@@ -44,14 +44,13 @@ final class DocFieldProcessorPerThread e
float docBoost;
int fieldGen;
final DocFieldProcessor docFieldProcessor;
- final FieldInfos fieldInfos;
final DocFieldConsumerPerThread consumer;
// Holds all fields seen in current doc
DocFieldProcessorPerField[] fields = new DocFieldProcessorPerField[1];
int fieldCount;
- // Hash table for all fields ever seen
+ // Hash table for all fields seen in current segment
DocFieldProcessorPerField[] fieldHash = new DocFieldProcessorPerField[2];
int hashMask = 1;
int totalFieldCount;
@@ -63,7 +62,6 @@ final class DocFieldProcessorPerThread e
public DocFieldProcessorPerThread(DocumentsWriterThreadState threadState, DocFieldProcessor docFieldProcessor) throws IOException {
this.docState = threadState.docState;
this.docFieldProcessor = docFieldProcessor;
- this.fieldInfos = docFieldProcessor.fieldInfos;
this.consumer = docFieldProcessor.consumer.addThread(this);
fieldsWriter = docFieldProcessor.fieldsWriter.addThread(docState);
}
@@ -78,6 +76,7 @@ final class DocFieldProcessorPerThread e
field = next;
}
}
+ doAfterFlush();
fieldsWriter.abort();
consumer.abort();
}
@@ -95,45 +94,15 @@ final class DocFieldProcessorPerThread e
return fields;
}
- /** If there are fields we've seen but did not see again
- * in the last run, then free them up. */
-
- void trimFields(SegmentWriteState state) {
-
- for(int i=0;i<fieldHash.length;i++) {
- DocFieldProcessorPerField perField = fieldHash[i];
- DocFieldProcessorPerField lastPerField = null;
-
- while (perField != null) {
-
- if (perField.lastGen == -1) {
-
- // This field was not seen since the previous
- // flush, so, free up its resources now
-
- // Unhash
- if (lastPerField == null)
- fieldHash[i] = perField.next;
- else
- lastPerField.next = perField.next;
-
- if (state.infoStream != null) {
- state.infoStream.println(" purge field=" + perField.fieldInfo.name);
+ /** In flush we reset the fieldHash to not maintain per-field state
+ * across segments */
+ @Override
+ void doAfterFlush() {
+ fieldHash = new DocFieldProcessorPerField[2];
+ hashMask = 1;
+ totalFieldCount = 0;
}
- totalFieldCount--;
-
- } else {
- // Reset
- perField.lastGen = -1;
- lastPerField = perField;
- }
-
- perField = perField.next;
- }
- }
- }
-
private void rehash() {
final int newHashSize = (fieldHash.length*2);
assert newHashSize > fieldHash.length;
@@ -158,7 +127,7 @@ final class DocFieldProcessorPerThread e
}
@Override
- public DocumentsWriter.DocWriter processDocument() throws IOException {
+ public DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos) throws IOException {
consumer.startDocument();
fieldsWriter.startDocument();
@@ -196,7 +165,7 @@ final class DocFieldProcessorPerThread e
// needs to be more "pluggable" such that if I want
// to have a new "thing" my Fields can do, I can
// easily add it
- FieldInfo fi = fieldInfos.add(fieldName, field.isIndexed(), field.isTermVectorStored(),
+ FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.isIndexed(), field.isTermVectorStored(),
field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
field.getOmitNorms(), false, field.getOmitTermFreqAndPositions(), field.docValuesType());
fp = new DocFieldProcessorPerField(this, fi);
@@ -206,11 +175,11 @@ final class DocFieldProcessorPerThread e
if (totalFieldCount >= fieldHash.length/2)
rehash();
- } else
- fp.fieldInfo.update(field.isIndexed(), field.isTermVectorStored(),
+ } else {
+ fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(),
field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
-
+ field.getOmitNorms(), false, field.getOmitTermFreqAndPositions(), field.docValuesType());
+ }
if (thisFieldGen != fp.lastGen) {
// First time we're seeing this field for this doc
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocInverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocInverter.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocInverter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocInverter.java Mon Mar 28 10:50:28 2011
@@ -40,13 +40,6 @@ final class DocInverter extends DocField
}
@Override
- void setFieldInfos(FieldInfos fieldInfos) {
- super.setFieldInfos(fieldInfos);
- consumer.setFieldInfos(fieldInfos);
- endConsumer.setFieldInfos(fieldInfos);
- }
-
- @Override
void flush(Map<DocFieldConsumerPerThread, Collection<DocFieldConsumerPerField>> threadsAndFields, SegmentWriteState state) throws IOException {
Map<InvertedDocConsumerPerThread,Collection<InvertedDocConsumerPerField>> childThreadsAndFields = new HashMap<InvertedDocConsumerPerThread,Collection<InvertedDocConsumerPerField>>();
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Mon Mar 28 10:50:28 2011
@@ -186,7 +186,6 @@ final class DocumentsWriter {
/**
* RAMFile buffer for DocWriters.
*/
- @SuppressWarnings("serial")
class PerDocBuffer extends RAMFile {
/**
@@ -266,34 +265,26 @@ final class DocumentsWriter {
// How much RAM we can use before flushing. This is 0 if
// we are flushing by doc count instead.
- private long ramBufferSize = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024);
- private long waitQueuePauseBytes = (long) (ramBufferSize*0.1);
- private long waitQueueResumeBytes = (long) (ramBufferSize*0.05);
-
- // If we've allocated 5% over our RAM budget, we then
- // free down to 95%
- private long freeLevel = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*0.95);
-
- // Flush @ this number of docs. If ramBufferSize is
- // non-zero we will flush by RAM usage instead.
- private int maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS;
+
+ private final IndexWriterConfig config;
private boolean closed;
- private final FieldInfos fieldInfos;
+ private FieldInfos fieldInfos;
private final BufferedDeletesStream bufferedDeletesStream;
private final IndexWriter.FlushControl flushControl;
- DocumentsWriter(Directory directory, IndexWriter writer, IndexingChain indexingChain, int maxThreadStates, FieldInfos fieldInfos, BufferedDeletesStream bufferedDeletesStream) throws IOException {
+ DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, IndexingChain indexingChain, FieldInfos fieldInfos,
+ BufferedDeletesStream bufferedDeletesStream) throws IOException {
this.directory = directory;
this.writer = writer;
- this.similarityProvider = writer.getConfig().getSimilarityProvider();
- this.maxThreadStates = maxThreadStates;
+ this.similarityProvider = config.getSimilarityProvider();
+ this.maxThreadStates = config.getMaxThreadStates();
this.fieldInfos = fieldInfos;
this.bufferedDeletesStream = bufferedDeletesStream;
flushControl = writer.flushControl;
-
- consumer = indexingChain.getChain(this);
+ consumer = config.getIndexingChain().getChain(this);
+ this.config = config;
}
// Buffer a specific docID for deletion. Currently only
@@ -350,10 +341,6 @@ final class DocumentsWriter {
return doFlush;
}
- public FieldInfos getFieldInfos() {
- return fieldInfos;
- }
-
/** If non-null, various details of indexing are printed
* here. */
synchronized void setInfoStream(PrintStream infoStream) {
@@ -363,45 +350,6 @@ final class DocumentsWriter {
}
}
- synchronized void setSimilarityProvider(SimilarityProvider similarity) {
- this.similarityProvider = similarity;
- for(int i=0;i<threadStates.length;i++) {
- threadStates[i].docState.similarityProvider = similarity;
- }
- }
-
- /** Set how much RAM we can use before flushing. */
- synchronized void setRAMBufferSizeMB(double mb) {
- if (mb == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
- ramBufferSize = IndexWriterConfig.DISABLE_AUTO_FLUSH;
- waitQueuePauseBytes = 4*1024*1024;
- waitQueueResumeBytes = 2*1024*1024;
- } else {
- ramBufferSize = (long) (mb*1024*1024);
- waitQueuePauseBytes = (long) (ramBufferSize*0.1);
- waitQueueResumeBytes = (long) (ramBufferSize*0.05);
- freeLevel = (long) (0.95 * ramBufferSize);
- }
- }
-
- synchronized double getRAMBufferSizeMB() {
- if (ramBufferSize == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
- return ramBufferSize;
- } else {
- return ramBufferSize/1024./1024.;
- }
- }
-
- /** Set max buffered docs, which means we will flush by
- * doc count instead of by RAM usage. */
- void setMaxBufferedDocs(int count) {
- maxBufferedDocs = count;
- }
-
- int getMaxBufferedDocs() {
- return maxBufferedDocs;
- }
-
/** Get current segment name we are writing. */
synchronized String getSegment() {
return segment;
@@ -482,9 +430,14 @@ final class DocumentsWriter {
private void doAfterFlush() throws IOException {
// All ThreadStates should be idle when we are called
assert allThreadsIdle();
+ for (DocumentsWriterThreadState threadState : threadStates) {
+ threadState.consumer.doAfterFlush();
+ }
+
threadBindings.clear();
waitQueue.reset();
segment = null;
+ fieldInfos = new FieldInfos(fieldInfos);
numDocs = 0;
nextDocID = 0;
bufferIsFull = false;
@@ -518,7 +471,7 @@ final class DocumentsWriter {
if (segmentInfos.size() > 0 || newSegment != null) {
final FrozenBufferedDeletes packet = new FrozenBufferedDeletes(pendingDeletes, delGen);
if (infoStream != null) {
- message("flush: push buffered deletes");
+ message("flush: push buffered deletes startSize=" + pendingDeletes.bytesUsed.get() + " frozenSize=" + packet.bytesUsed);
}
bufferedDeletesStream.push(packet);
if (infoStream != null) {
@@ -585,9 +538,8 @@ final class DocumentsWriter {
if (infoStream != null) {
message("flush postings as segment " + segment + " numDocs=" + numDocs);
}
-
- final SegmentWriteState flushState = segWriteState();
-
+
+ final SegmentWriteState flushState = segWriteState(true);
// Apply delete-by-docID now (delete-byDocID only
// happens when an exception is hit processing that
// doc, eg if analyzer has some problem w/ the text):
@@ -600,7 +552,7 @@ final class DocumentsWriter {
pendingDeletes.docIDs.clear();
}
- newSegment = new SegmentInfo(segment, numDocs, directory, false, fieldInfos.hasProx(), flushState.segmentCodecs, false);
+ newSegment = new SegmentInfo(segment, numDocs, directory, false, fieldInfos.hasProx(), flushState.segmentCodecs, false, fieldInfos);
Collection<DocConsumerPerThread> threads = new HashSet<DocConsumerPerThread>();
for (DocumentsWriterThreadState threadState : threadStates) {
@@ -703,10 +655,11 @@ final class DocumentsWriter {
return newSegment;
}
- SegmentWriteState segWriteState() {
+ SegmentWriteState segWriteState(boolean flush) {
return new SegmentWriteState(infoStream, directory, segment, fieldInfos,
numDocs, writer.getConfig().getTermIndexInterval(),
- SegmentCodecs.build(fieldInfos, writer.codecs), pendingDeletes, bytesUsed);
+ fieldInfos.buildSegmentCodecs(flush),
+ pendingDeletes, bytesUsed);
}
synchronized void close() {
@@ -800,7 +753,7 @@ final class DocumentsWriter {
// work
final DocWriter perDoc;
try {
- perDoc = state.consumer.processDocument();
+ perDoc = state.consumer.processDocument(fieldInfos);
} finally {
docState.clear();
}
@@ -1028,6 +981,14 @@ final class DocumentsWriter {
deletesRAMUsed = bufferedDeletesStream.bytesUsed();
+ final long ramBufferSize;
+ final double mb = config.getRAMBufferSizeMB();
+ if (mb == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+ ramBufferSize = IndexWriterConfig.DISABLE_AUTO_FLUSH;
+ } else {
+ ramBufferSize = (long) (mb*1024*1024);
+ }
+
synchronized(this) {
if (ramBufferSize == IndexWriterConfig.DISABLE_AUTO_FLUSH || bufferIsFull) {
return;
@@ -1056,6 +1017,8 @@ final class DocumentsWriter {
boolean any = true;
+ final long freeLevel = (long) (0.95 * ramBufferSize);
+
while(bytesUsed()+deletesRAMUsed > freeLevel) {
synchronized(this) {
@@ -1121,10 +1084,24 @@ final class DocumentsWriter {
}
synchronized boolean doResume() {
+ final double mb = config.getRAMBufferSizeMB();
+ final long waitQueueResumeBytes;
+ if (mb == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+ waitQueueResumeBytes = 2*1024*1024;
+ } else {
+ waitQueueResumeBytes = (long) (mb*1024*1024*0.05);
+ }
return waitingBytes <= waitQueueResumeBytes;
}
synchronized boolean doPause() {
+ final double mb = config.getRAMBufferSizeMB();
+ final long waitQueuePauseBytes;
+ if (mb == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+ waitQueuePauseBytes = 4*1024*1024;
+ } else {
+ waitQueuePauseBytes = (long) (mb*1024*1024*0.1);
+ }
return waitingBytes > waitQueuePauseBytes;
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java Mon Mar 28 10:50:28 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.values.Ty
/** @lucene.experimental */
public final class FieldInfo {
+ public static final int UNASSIGNED_CODEC_ID = -1;
public String name;
public boolean isIndexed;
public int number;
@@ -36,14 +37,15 @@ public final class FieldInfo {
public boolean omitTermFreqAndPositions;
public boolean storePayloads; // whether this field stores payloads together with term positions
- int codecId = 0; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field
+ private int codecId = UNASSIGNED_CODEC_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field
FieldInfo(String na, boolean tk, int nu, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
- boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) {
+ boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions, Type docValues) {
name = na;
isIndexed = tk;
number = nu;
+ this.docValues = docValues;
if (isIndexed) {
this.storeTermVector = storeTermVector;
this.storeOffsetWithTermVector = storeOffsetWithTermVector;
@@ -61,12 +63,24 @@ public final class FieldInfo {
}
}
+ void setCodecId(int codecId) {
+ assert this.codecId == UNASSIGNED_CODEC_ID : "CodecId can only be set once.";
+ this.codecId = codecId;
+ }
+
+ public int getCodecId() {
+ return codecId;
+ }
+
@Override
public Object clone() {
- return new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector,
- storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
+ FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector,
+ storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValues);
+ clone.codecId = this.codecId;
+ return clone;
}
+ // should only be called by FieldInfos#addOrUpdate
void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector,
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) {
if (this.isIndexed != isIndexed) {
@@ -95,12 +109,8 @@ public final class FieldInfo {
}
void setDocValues(Type v) {
- if (docValues != null) {
- if (docValues != v) {
- throw new IllegalArgumentException("indexValues is already set to " + docValues + "; cannot change to " + v);
- }
- } else{
- docValues = v;
+ if (docValues == null) {
+ docValues = v;
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java Mon Mar 28 10:50:28 2011
@@ -17,17 +17,26 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.values.Type;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.StringHelper;
-import java.io.IOException;
-import java.util.*;
-
/** Access to the Fieldable Info file that describes document fields and whether or
* not they are indexed. Each segment has a separate Fieldable Info file. Objects
* of this class are thread-safe for multiple readers, but only one thread can
@@ -35,8 +44,160 @@ import java.util.*;
* accessing this object.
* @lucene.experimental
*/
-public final class FieldInfos {
+public final class FieldInfos implements Iterable<FieldInfo> {
+ static final class FieldNumberBiMap {
+
+ final static String CODEC_NAME = "GLOBAL_FIELD_MAP";
+
+ // Initial format
+ private static final int VERSION_START = 0;
+
+ private static final int VERSION_CURRENT = VERSION_START;
+
+ private final Map<Integer,String> numberToName;
+ private final Map<String,Integer> nameToNumber;
+ private int lowestUnassignedFieldNumber = -1;
+ private long lastVersion = 0;
+ private long version = 0;
+
+ FieldNumberBiMap() {
+ this.nameToNumber = new HashMap<String, Integer>();
+ this.numberToName = new HashMap<Integer, String>();
+ }
+
+ /**
+ * Returns the global field number for the given field name. If the name
+ * does not exist yet it tries to add it with the given preferred field
+ * number assigned if possible otherwise the first unassigned field number
+ * is used as the field number.
+ */
+ synchronized int addOrGet(String fieldName, int preferredFieldNumber) {
+ Integer fieldNumber = nameToNumber.get(fieldName);
+ if (fieldNumber == null) {
+ final Integer preferredBoxed = Integer.valueOf(preferredFieldNumber);
+
+ if (preferredFieldNumber != -1 && !numberToName.containsKey(preferredBoxed)) {
+ // cool - we can use this number globally
+ fieldNumber = preferredBoxed;
+ } else {
+ // find a new FieldNumber
+ while (numberToName.containsKey(++lowestUnassignedFieldNumber)) {
+ // might not be up to date - lets do the work once needed
+ }
+ fieldNumber = lowestUnassignedFieldNumber;
+ }
+
+ version++;
+ numberToName.put(fieldNumber, fieldName);
+ nameToNumber.put(fieldName, fieldNumber);
+
+ }
+
+ return fieldNumber.intValue();
+ }
+ /**
+ * Sets the given field number and name if not yet set.
+ */
+ synchronized void setIfNotSet(int fieldNumber, String fieldName) {
+ final Integer boxedFieldNumber = Integer.valueOf(fieldNumber);
+ if (!numberToName.containsKey(boxedFieldNumber)
+ && !nameToNumber.containsKey(fieldName)) {
+ version++;
+ numberToName.put(boxedFieldNumber, fieldName);
+ nameToNumber.put(fieldName, boxedFieldNumber);
+ } else {
+ assert containsConsistent(boxedFieldNumber, fieldName);
+ }
+ }
+
+ /**
+ * Writes this {@link FieldNumberBiMap} to the given output and returns its
+ * version.
+ */
+ public synchronized long write(IndexOutput output) throws IOException{
+ Set<Entry<String, Integer>> entrySet = nameToNumber.entrySet();
+ CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT);
+ output.writeVInt(entrySet.size());
+ for (Entry<String, Integer> entry : entrySet) {
+ output.writeVInt(entry.getValue().intValue());
+ output.writeString(entry.getKey());
+ }
+ return version;
+ }
+
+ /**
+ * Reads the {@link FieldNumberBiMap} from the given input and resets the
+ * version to 0.
+ */
+ public synchronized void read(IndexInput input) throws IOException{
+ CodecUtil.checkHeader(input, CODEC_NAME,
+ VERSION_START,
+ VERSION_CURRENT);
+ final int size = input.readVInt();
+ for (int i = 0; i < size; i++) {
+ final int num = input.readVInt();
+ final String name = input.readString();
+ setIfNotSet(num, name);
+ }
+ version = lastVersion = 0;
+ }
+
+ /**
+ * Returns a new {@link FieldInfos} instance with this as the global field
+ * map
+ *
+ * @return a new {@link FieldInfos} instance with this as the global field
+ * map
+ */
+ public FieldInfos newFieldInfos(SegmentCodecsBuilder segmentCodecsBuilder) {
+ return new FieldInfos(this, segmentCodecsBuilder);
+ }
+
+ /**
+ * Returns <code>true</code> iff the last committed version differs from the
+ * current version, otherwise <code>false</code>
+ *
+ * @return <code>true</code> iff the last committed version differs from the
+ * current version, otherwise <code>false</code>
+ */
+ public synchronized boolean isDirty() {
+ return lastVersion != version;
+ }
+
+ /**
+ * commits the given version if the given version is greater than the previous committed version
+ *
+ * @param version
+ * the version to commit
+ * @return <code>true</code> iff the version was successfully committed otherwise <code>false</code>
+ * @see #write(IndexOutput)
+ */
+ public synchronized boolean commitLastVersion(long version) {
+ if (version > lastVersion) {
+ lastVersion = version;
+ return true;
+ }
+ return false;
+ }
+
+ // just for testing
+ Set<Entry<String, Integer>> entries() {
+ return new HashSet<Entry<String, Integer>>(nameToNumber.entrySet());
+ }
+
+ // used by assert
+ boolean containsConsistent(Integer number, String name) {
+ return name.equals(numberToName.get(number))
+ && number.equals(nameToNumber.get(name));
+ }
+ }
+
+ private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
+ private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
+ private final FieldNumberBiMap globalFieldNumbers;
+ private final SegmentCodecsBuilder segmentCodecsBuilder;
+
// First used in 2.9; prior to 2.9 there was no format header
public static final int FORMAT_START = -2;
public static final int FORMAT_PER_FIELD_CODEC = -3;
@@ -56,22 +217,54 @@ public final class FieldInfos {
static final byte OMIT_NORMS = 0x10;
static final byte STORE_PAYLOADS = 0x20;
static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40;
-
- private final ArrayList<FieldInfo> byNumber = new ArrayList<FieldInfo>();
- private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
+
private int format;
+ /**
+ * Creates a new {@link FieldInfos} instance with a private
+ * {@link FieldNumberBiMap} and a default {@link SegmentCodecsBuilder}
+ * initialized with {@link CodecProvider#getDefault()}.
+ * <p>
+ * Note: this ctor should not be used during indexing use
+ * {@link FieldInfos#FieldInfos(FieldInfos)} or
+ * {@link FieldInfos#FieldInfos(FieldNumberBiMap)} instead.
+ */
public FieldInfos() {
+ this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault()));
+ }
+
+ /**
+ * Creates a new {@link FieldInfo} instance from the given instance. If the given instance is
+ * read-only this instance will be read-only too.
+ *
+ * @see #isReadOnly()
+ */
+ FieldInfos(FieldInfos other) {
+ this(other.globalFieldNumbers, other.segmentCodecsBuilder);
+ }
+
+ /**
+ * Creates a new FieldInfos instance with the given {@link FieldNumberBiMap}.
+ * If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only.
+ * @see #isReadOnly()
+ */
+ FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentCodecsBuilder segmentCodecsBuilder) {
+ this.globalFieldNumbers = globalFieldNumbers;
+ this.segmentCodecsBuilder = segmentCodecsBuilder;
}
/**
* Construct a FieldInfos object using the directory and the name of the file
- * IndexInput
+ * IndexInput.
+ * <p>
+ * Note: The created instance will be read-only
+ *
* @param d The directory to open the IndexInput from
* @param name The name of the file to open the IndexInput from in the Directory
* @throws IOException
*/
public FieldInfos(Directory d, String name) throws IOException {
+ this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only
IndexInput input = d.openInput(name);
try {
read(input, name);
@@ -79,36 +272,45 @@ public final class FieldInfos {
input.close();
}
}
+
+ /**
+ * adds the given field to this FieldInfos name / number mapping. The given FI
+ * must be present in the global field number mapping before this method it
+ * called
+ */
+ private void putInternal(FieldInfo fi) {
+ assert !byNumber.containsKey(fi.number);
+ assert !byName.containsKey(fi.name);
+ assert globalFieldNumbers == null || globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name);
+ byNumber.put(fi.number, fi);
+ byName.put(fi.name, fi);
+ }
+
+ private int nextFieldNumber(String name, int preferredFieldNumber) {
+ // get a global number for this field
+ final int fieldNumber = globalFieldNumbers.addOrGet(name,
+ preferredFieldNumber);
+ assert byNumber.get(fieldNumber) == null : "field number " + fieldNumber
+ + " already taken";
+ return fieldNumber;
+ }
/**
* Returns a deep clone of this FieldInfos instance.
*/
@Override
synchronized public Object clone() {
- FieldInfos fis = new FieldInfos();
- final int numField = byNumber.size();
- for(int i=0;i<numField;i++) {
- FieldInfo fi = (FieldInfo) ( byNumber.get(i)).clone();
- fis.byNumber.add(fi);
- fis.byName.put(fi.name, fi);
+ FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentCodecsBuilder);
+ for (FieldInfo fi : this) {
+ FieldInfo clone = (FieldInfo) (fi).clone();
+ fis.putInternal(clone);
}
return fis;
}
- /** Adds field info for a Document. */
- synchronized public void add(Document doc) {
- List<Fieldable> fields = doc.getFields();
- for (Fieldable field : fields) {
- add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
- field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions(), field.docValuesType());
- }
- }
-
/** Returns true if any fields do not omitTermFreqAndPositions */
public boolean hasProx() {
- final int numFields = byNumber.size();
- for(int i=0;i<numFields;i++) {
- final FieldInfo fi = fieldInfo(i);
+ for (FieldInfo fi : this) {
if (fi.isIndexed && !fi.omitTermFreqAndPositions) {
return true;
}
@@ -117,17 +319,17 @@ public final class FieldInfos {
}
/**
- * Add fields that are indexed. Whether they have termvectors has to be specified.
+ * Adds or updates fields that are indexed. Whether they have termvectors has to be specified.
*
* @param names The names of the fields
* @param storeTermVectors Whether the fields store term vectors or not
* @param storePositionWithTermVector true if positions should be stored.
* @param storeOffsetWithTermVector true if offsets should be stored
*/
- synchronized public void addIndexed(Collection<String> names, boolean storeTermVectors, boolean storePositionWithTermVector,
+ synchronized public void addOrUpdateIndexed(Collection<String> names, boolean storeTermVectors, boolean storePositionWithTermVector,
boolean storeOffsetWithTermVector) {
for (String name : names) {
- add(name, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector);
+ addOrUpdate(name, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector);
}
}
@@ -137,11 +339,11 @@ public final class FieldInfos {
* @param names The names of the fields
* @param isIndexed Whether the fields are indexed or not
*
- * @see #add(String, boolean)
+ * @see #addOrUpdate(String, boolean)
*/
- synchronized public void add(Collection<String> names, boolean isIndexed) {
+ synchronized public void addOrUpdate(Collection<String> names, boolean isIndexed) {
for (String name : names) {
- add(name, isIndexed);
+ addOrUpdate(name, isIndexed);
}
}
@@ -150,10 +352,10 @@ public final class FieldInfos {
*
* @param name The name of the Fieldable
* @param isIndexed true if the field is indexed
- * @see #add(String, boolean, boolean, boolean, boolean)
+ * @see #addOrUpdate(String, boolean, boolean, boolean, boolean)
*/
- synchronized public void add(String name, boolean isIndexed) {
- add(name, isIndexed, false, false, false, false);
+ synchronized public void addOrUpdate(String name, boolean isIndexed) {
+ addOrUpdate(name, isIndexed, false, false, false, false);
}
/**
@@ -163,8 +365,8 @@ public final class FieldInfos {
* @param isIndexed true if the field is indexed
* @param storeTermVector true if the term vector should be stored
*/
- synchronized public void add(String name, boolean isIndexed, boolean storeTermVector){
- add(name, isIndexed, storeTermVector, false, false, false);
+ synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector){
+ addOrUpdate(name, isIndexed, storeTermVector, false, false, false);
}
/** If the field is not yet known, adds it. If it is known, checks to make
@@ -178,10 +380,10 @@ public final class FieldInfos {
* @param storePositionWithTermVector true if the term vector with positions should be stored
* @param storeOffsetWithTermVector true if the term vector with offsets should be stored
*/
- synchronized public void add(String name, boolean isIndexed, boolean storeTermVector,
+ synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector) {
- add(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false);
+ addOrUpdate(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false);
}
/** If the field is not yet known, adds it. If it is known, checks to make
@@ -196,9 +398,9 @@ public final class FieldInfos {
* @param storeOffsetWithTermVector true if the term vector with offsets should be stored
* @param omitNorms true if the norms for the indexed field should be omitted
*/
- synchronized public void add(String name, boolean isIndexed, boolean storeTermVector,
+ synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms) {
- add(name, isIndexed, storeTermVector, storePositionWithTermVector,
+ addOrUpdate(name, isIndexed, storeTermVector, storePositionWithTermVector,
storeOffsetWithTermVector, omitNorms, false, false, null);
}
@@ -216,35 +418,56 @@ public final class FieldInfos {
* @param storePayloads true if payloads should be stored for this field
* @param omitTermFreqAndPositions true if term freqs should be omitted for this field
*/
- synchronized public FieldInfo add(String name, boolean isIndexed, boolean storeTermVector,
+ synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions, Type docValues) {
+ return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, storePositionWithTermVector,
+ storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValues);
+ }
+
+ synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
+ boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
+ boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions, Type docValues) {
+ if (globalFieldNumbers == null) {
+ throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
+ }
+ assert segmentCodecsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only";
FieldInfo fi = fieldInfo(name);
if (fi == null) {
- return addInternal(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValues);
+ final int fieldNumber = nextFieldNumber(name, preferredFieldNumber);
+ fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValues);
} else {
fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
fi.setDocValues(docValues);
}
+ if ((fi.isIndexed || fi.hasDocValues()) && fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) {
+ segmentCodecsBuilder.tryAddAndSet(fi);
+ }
return fi;
}
synchronized public FieldInfo add(FieldInfo fi) {
- return add(fi.name, fi.isIndexed, fi.storeTermVector,
+ // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
+ return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector,
fi.storePositionWithTermVector, fi.storeOffsetWithTermVector,
fi.omitNorms, fi.storePayloads,
fi.omitTermFreqAndPositions, fi.docValues);
}
-
- private FieldInfo addInternal(String name, boolean isIndexed,
+
+ /*
+ * NOTE: if you call this method from a public method make sure you check if we are modifiable and throw an exception otherwise
+ */
+ private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
boolean storeTermVector, boolean storePositionWithTermVector,
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions, Type docValuesType) {
+ // don't check modifiable here since we use that to initially build up FIs
name = StringHelper.intern(name);
- FieldInfo fi = new FieldInfo(name, isIndexed, byNumber.size(), storeTermVector, storePositionWithTermVector,
- storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
- fi.setDocValues(docValuesType);
- byNumber.add(fi);
- byName.put(name, fi);
+ if (globalFieldNumbers != null) {
+ globalFieldNumbers.setIfNotSet(fieldNumber, name);
+ }
+ final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, storePositionWithTermVector,
+ storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
+ putInternal(fi);
return fi;
}
@@ -254,7 +477,7 @@ public final class FieldInfos {
}
public FieldInfo fieldInfo(String fieldName) {
- return byName.get(fieldName);
+ return byName.get(fieldName);
}
/**
@@ -265,8 +488,8 @@ public final class FieldInfos {
* with the given number doesn't exist.
*/
public String fieldName(int fieldNumber) {
- FieldInfo fi = fieldInfo(fieldNumber);
- return (fi != null) ? fi.name : "";
+ FieldInfo fi = fieldInfo(fieldNumber);
+ return (fi != null) ? fi.name : "";
}
/**
@@ -279,13 +502,18 @@ public final class FieldInfos {
return (fieldNumber >= 0) ? byNumber.get(fieldNumber) : null;
}
+ public Iterator<FieldInfo> iterator() {
+ return byNumber.values().iterator();
+ }
+
public int size() {
+ assert byNumber.size() == byName.size();
return byNumber.size();
}
public boolean hasVectors() {
- for (int i = 0; i < size(); i++) {
- if (fieldInfo(i).storeTermVector) {
+ for (FieldInfo fi : this) {
+ if (fi.storeTermVector) {
return true;
}
}
@@ -293,13 +521,29 @@ public final class FieldInfos {
}
public boolean hasNorms() {
- for (int i = 0; i < size(); i++) {
- if (!fieldInfo(i).omitNorms) {
+ for (FieldInfo fi : this) {
+ if (!fi.omitNorms) {
return true;
}
}
return false;
}
+
+ /**
+ * Builds the {@link SegmentCodecs} mapping for this {@link FieldInfos} instance.
+ * @param clearBuilder <code>true</code> iff the internal {@link SegmentCodecsBuilder} must be cleared otherwise <code>false</code>
+ */
+ public SegmentCodecs buildSegmentCodecs(boolean clearBuilder) {
+ if (globalFieldNumbers == null) {
+ throw new IllegalStateException("FieldInfos are read-only no SegmentCodecs available");
+ }
+ assert segmentCodecsBuilder != null;
+ final SegmentCodecs segmentCodecs = segmentCodecsBuilder.build();
+ if (clearBuilder) {
+ segmentCodecsBuilder.clear();
+ }
+ return segmentCodecs;
+ }
public void write(Directory d, String name) throws IOException {
IndexOutput output = d.createOutput(name);
@@ -309,12 +553,21 @@ public final class FieldInfos {
output.close();
}
}
+
+ /**
+ * Returns <code>true</code> iff this instance is not backed by a
+ * {@link FieldNumberBiMap}. Instances read from a directory via
+ * {@link FieldInfos#FieldInfos(Directory, String)} will always be read-only
+ * since no {@link FieldNumberBiMap} is supplied, otherwise <code>false</code>.
+ */
+ public final boolean isReadOnly() {
+ return globalFieldNumbers == null;
+ }
public void write(IndexOutput output) throws IOException {
output.writeVInt(FORMAT_CURRENT);
output.writeVInt(size());
- for (int i = 0; i < size(); i++) {
- FieldInfo fi = fieldInfo(i);
+ for (FieldInfo fi : this) {
byte bits = 0x0;
if (fi.isIndexed) bits |= IS_INDEXED;
if (fi.storeTermVector) bits |= STORE_TERMVECTOR;
@@ -324,7 +577,8 @@ public final class FieldInfos {
if (fi.storePayloads) bits |= STORE_PAYLOADS;
if (fi.omitTermFreqAndPositions) bits |= OMIT_TERM_FREQ_AND_POSITIONS;
output.writeString(fi.name);
- output.writeInt(fi.codecId);
+ output.writeInt(fi.number);
+ output.writeInt(fi.getCodecId());
output.writeByte(bits);
final byte b;
@@ -383,6 +637,7 @@ public final class FieldInfos {
for (int i = 0; i < size; i++) {
String name = StringHelper.intern(input.readString());
// if this is a previous format codec 0 will be preflex!
+ final int fieldNumber = format <= FORMAT_PER_FIELD_CODEC? input.readInt():i;
final int codecId = format <= FORMAT_PER_FIELD_CODEC? input.readInt():0;
byte bits = input.readByte();
boolean isIndexed = (bits & IS_INDEXED) != 0;
@@ -430,9 +685,8 @@ public final class FieldInfos {
throw new IllegalStateException("unhandled indexValues type " + b);
}
}
- final FieldInfo fi = addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
-
- fi.codecId = codecId;
+ final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
+ addInternal.setCodecId(codecId);
}
if (input.getFilePointer() != input.length()) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsWriter.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsWriter.java Mon Mar 28 10:50:28 2011
@@ -45,14 +45,12 @@ final class FieldsWriter {
// If null - we were supplied with streams, if notnull - we manage them ourselves
private Directory directory;
private String segment;
- private FieldInfos fieldInfos;
private IndexOutput fieldsStream;
private IndexOutput indexStream;
- FieldsWriter(Directory directory, String segment, FieldInfos fn) throws IOException {
+ FieldsWriter(Directory directory, String segment) throws IOException {
this.directory = directory;
this.segment = segment;
- fieldInfos = fn;
boolean success = false;
try {
@@ -70,10 +68,9 @@ final class FieldsWriter {
}
}
- FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) {
+ FieldsWriter(IndexOutput fdx, IndexOutput fdt) {
directory = null;
segment = null;
- fieldInfos = fn;
fieldsStream = fdt;
indexStream = fdx;
}
@@ -166,7 +163,7 @@ final class FieldsWriter {
assert fieldsStream.getFilePointer() == position;
}
- final void addDocument(Document doc) throws IOException {
+ final void addDocument(Document doc, FieldInfos fieldInfos) throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
int storedCount = 0;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Mon Mar 28 10:50:28 2011
@@ -78,7 +78,10 @@ public final class IndexFileNames {
/** Extension of separate norms */
public static final String SEPARATE_NORMS_EXTENSION = "s";
-
+
+ /** Extension of global field numbers */
+ public static final String GLOBAL_FIELD_NUM_MAP_EXTENSION = "fnx";
+
/**
* This array contains all filename extensions used by
* Lucene's index files, with one exception, namely the
@@ -98,6 +101,7 @@ public final class IndexFileNames {
GEN_EXTENSION,
NORMS_EXTENSION,
COMPOUND_FILE_STORE_EXTENSION,
+ GLOBAL_FIELD_NUM_MAP_EXTENSION,
};
public static final String[] STORE_INDEX_EXTENSIONS = new String[] {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1086181&r1=1086180&r2=1086181&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java Mon Mar 28 10:50:28 2011
@@ -1000,7 +1000,8 @@ public abstract class IndexReader implem
}
/** Returns the byte-encoded normalization factor for the named field of
- * every document. This is used by the search code to score documents.
+ * every document. This is used by the search code to score documents.
+ * Returns null if norms were not indexed for this field.
*
* @see org.apache.lucene.document.Field#setBoost(float)
*/