You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/08/17 01:24:21 UTC
svn commit: r986160 [3/3] - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/ lucene/backwards/src/test/org/apache/lucene/analysis/
lucene/backwards/src/test/org/apache/lucene/document/
lucene/backwards/src/test/org/apache/lucene/inde...
Propchange: lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Aug 16 23:24:19 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/solr/src/webapp/web/admin:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,932998-932999,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940556,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,944516,944528,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950125,950207,950458,950613,950723,950835,951126,951355,951397,951521,953628,953886,954336,955547,955613,955615,955796-955797,955809-955996,956097,956125,956173,956316,956715,957465,957520,957634,957707,960367,960374,961821,961941,
962555,962714,963372,963654,963720,963781,963873,963888,963906,963909,963920,964019,964054,964267,964306,964312,964430,964459,964720,964753,964832,964856,965222,965230,965327,965330,965585,965938,966354,966878,967080,979049,979453,979807,979809,980369,980428,980436,980501,980555,980592,980909,980911,980917,981265,981550,981598,981650,981661,981690,981857,981936,982073,982084,982201,982725,982824,983100,983212,983216,983313,983495,983500,983530,983632,983778,984187,984202,984232,984510,984968,985453,985455,985672,985875
+/lucene/dev/trunk/solr/src/webapp/web/admin:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,932998-932999,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940556,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,944516,944528,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950125,950207,950458,950613,950723,950835,951126,951355,951397,951521,953628,953886,954336,955547,955613,955615,955796-955797,955809-955996,956097,956125,956173,956316,956715,957465,957520,957634,957707,960367,960374,961821,961941,
962555,962714,963372,963654,963720,963781,963873,963888,963906,963909,963920,964019,964054,964267,964306,964312,964430,964459,964720,964753,964832,964856,965222,965230,965327,965330,965585,965938,966354,966878,967080,979049,979453,979807,979809,980369,980428,980436,980501,980555,980592,980909,980911,980917,981265,981550,981598,981650,981661,981690,981857,981936,982073,982084,982201,982725,982824,983100,983212,983216,983313,983495,983500,983530,983632,983778,984187,984202,984232,984510,984968,985453,985455,985672,985875,986158
/lucene/java/branches/lucene_2_4/solr/src/webapp/web/admin:748824
/lucene/java/branches/lucene_2_9/solr/src/webapp/web/admin:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr/src/webapp/web/admin:818601-821336
Modified: lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/analysis.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/analysis.jsp?rev=986160&r1=986159&r2=986160&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/analysis.jsp (original)
+++ lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/analysis.jsp Mon Aug 16 23:24:19 2010
@@ -16,7 +16,8 @@
limitations under the License.
--%>
<%@ page import="org.apache.lucene.analysis.Analyzer,
- org.apache.lucene.analysis.Token,
+ org.apache.lucene.util.AttributeSource,
+ org.apache.lucene.util.Attribute,
org.apache.lucene.analysis.TokenStream,
org.apache.lucene.index.Payload,
org.apache.lucene.analysis.CharReader,
@@ -152,9 +153,9 @@
Analyzer analyzer = field.getType().getQueryAnalyzer();
TokenStream tstream = analyzer.reusableTokenStream(field.getName(),reader);
tstream.reset();
- List<Token> tokens = getTokens(tstream);
+ List<AttributeSource> tokens = getTokens(tstream);
matches = new HashSet<Tok>();
- for (Token t : tokens) { matches.add( new Tok(t,0)); }
+ for (AttributeSource t : tokens) { matches.add( new Tok(t,0)); }
}
if (val!="") {
@@ -199,7 +200,7 @@
}
TokenStream tstream = tfac.create(tchain.charStream(new StringReader(val)));
- List<Token> tokens = getTokens(tstream);
+ List<AttributeSource> tokens = getTokens(tstream);
if (verbose) {
writeHeader(out, tfac.getClass(), tfac.getArgs());
}
@@ -211,24 +212,16 @@
writeHeader(out, filtfac.getClass(), filtfac.getArgs());
}
- final Iterator<Token> iter = tokens.iterator();
- tstream = filtfac.create( new TokenStream() {
- CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- OffsetAttribute offsetAtt = addAttribute (OffsetAttribute.class);
- TypeAttribute typeAtt = addAttribute (TypeAttribute.class);
- FlagsAttribute flagsAtt = addAttribute (FlagsAttribute.class);
- PayloadAttribute payloadAtt = addAttribute (PayloadAttribute.class);
- PositionIncrementAttribute posIncAtt = addAttribute (PositionIncrementAttribute.class);
+ final Iterator<AttributeSource> iter = tokens.iterator();
+ tstream = filtfac.create( new TokenStream(tstream.getAttributeFactory()) {
public boolean incrementToken() throws IOException {
if (iter.hasNext()) {
- Token token = iter.next();
- termAtt.copyBuffer(token.buffer(), 0, token.length());
- offsetAtt.setOffset(token.startOffset(), token.endOffset());
- typeAtt.setType(token.type());
- flagsAtt.setFlags(token.getFlags());
- posIncAtt.setPositionIncrement(token.getPositionIncrement());
- payloadAtt.setPayload(token.getPayload());
+ AttributeSource token = iter.next();
+ Iterator<Class<? extends Attribute>> atts = token.getAttributeClassesIterator();
+ while (atts.hasNext()) // make sure all att impls in the token exist here
+ addAttribute(atts.next());
+ token.copyTo(this);
return true;
} else {
return false;
@@ -244,7 +237,7 @@
} else {
TokenStream tstream = analyzer.reusableTokenStream(field.getName(),new StringReader(val));
tstream.reset();
- List<Token> tokens = getTokens(tstream);
+ List<AttributeSource> tokens = getTokens(tstream);
if (verbose) {
writeHeader(out, analyzer.getClass(), new HashMap<String,String>());
}
@@ -253,27 +246,14 @@
}
- static List<Token> getTokens(TokenStream tstream) throws IOException {
- List<Token> tokens = new ArrayList<Token>();
- CharTermAttribute termAtt = tstream.addAttribute(CharTermAttribute.class);
- OffsetAttribute offsetAtt = tstream.addAttribute (OffsetAttribute.class);
- TypeAttribute typeAtt = tstream.addAttribute (TypeAttribute.class);
- FlagsAttribute flagsAtt = tstream.addAttribute (FlagsAttribute.class);
- PayloadAttribute payloadAtt = tstream.addAttribute (PayloadAttribute.class);
- PositionIncrementAttribute posIncAtt = tstream.addAttribute (PositionIncrementAttribute.class);
+ static List<AttributeSource> getTokens(TokenStream tstream) throws IOException {
+ List<AttributeSource> tokens = new ArrayList<AttributeSource>();
while (true) {
if (!tstream.incrementToken())
break;
else {
- Token token = new Token();
- token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
- token.setType(typeAtt.type());
- token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
- token.setPayload(payloadAtt.getPayload());
- token.setFlags(flagsAtt.getFlags());
- token.setPositionIncrement(posIncAtt.getPositionIncrement());
- tokens.add(token);
+ tokens.add(tstream.cloneAttributes());
}
}
return tokens;
@@ -281,9 +261,9 @@
private static class Tok {
- Token token;
+ AttributeSource token;
int pos;
- Tok(Token token, int pos) {
+ Tok(AttributeSource token, int pos) {
this.token=token;
this.pos=pos;
}
@@ -297,6 +277,9 @@
public String toString() {
return token.toString();
}
+ public String toPrintableString() {
+ return token.addAttribute(CharTermAttribute.class).toString();
+ }
}
private static interface ToStr {
@@ -368,7 +351,7 @@
// readable, raw, pos, type, start/end
- static void writeTokens(JspWriter out, List<Token> tokens, final FieldType ft, boolean verbose, Set<Tok> match) throws IOException {
+ static void writeTokens(JspWriter out, List<AttributeSource> tokens, final FieldType ft, boolean verbose, Set<Tok> match) throws IOException {
// Use a map to tell what tokens are in what positions
// because some tokenizers/filters may do funky stuff with
@@ -376,12 +359,12 @@
HashMap<Integer,List<Tok>> map = new HashMap<Integer,List<Tok>>();
boolean needRaw=false;
int pos=0;
- for (Token t : tokens) {
+ for (AttributeSource t : tokens) {
if (!t.toString().equals(ft.indexedToReadable(t.toString()))) {
needRaw=true;
}
- pos += t.getPositionIncrement();
+ pos += t.addAttribute(PositionIncrementAttribute.class).getPositionIncrement();
List lst = map.get(pos);
if (lst==null) {
lst = new ArrayList(1);
@@ -426,7 +409,7 @@
printRow(out,"term text", arr, new ToStr() {
public String toStr(Object o) {
- return ft.indexedToReadable( ((Tok)o).token.toString() );
+ return ft.indexedToReadable( ((Tok)o).toPrintableString() );
}
}
,true
@@ -438,7 +421,7 @@
printRow(out,"raw text", arr, new ToStr() {
public String toStr(Object o) {
// page is UTF-8, so anything goes.
- return ((Tok)o).token.toString();
+ return ((Tok)o).toPrintableString();
}
}
,true
@@ -450,7 +433,7 @@
if (verbose) {
printRow(out,"term type", arr, new ToStr() {
public String toStr(Object o) {
- String tt = ((Tok)o).token.type();
+ String tt = ((Tok)o).token.addAttribute(TypeAttribute.class).type();
if (tt == null) {
return "null";
} else {
@@ -467,8 +450,8 @@
if (verbose) {
printRow(out,"source start,end", arr, new ToStr() {
public String toStr(Object o) {
- Token t = ((Tok)o).token;
- return Integer.toString(t.startOffset()) + ',' + t.endOffset() ;
+ AttributeSource t = ((Tok)o).token;
+ return Integer.toString(t.addAttribute(OffsetAttribute.class).startOffset()) + ',' + t.addAttribute(OffsetAttribute.class).endOffset() ;
}
}
,true
@@ -480,8 +463,8 @@
if (verbose) {
printRow(out,"payload", arr, new ToStr() {
public String toStr(Object o) {
- Token t = ((Tok)o).token;
- Payload p = t.getPayload();
+ AttributeSource t = ((Tok)o).token;
+ Payload p = t.addAttribute(PayloadAttribute.class).getPayload();
if( null != p ) {
BigInteger bi = new BigInteger( p.getData() );
String ret = bi.toString( 16 );