You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2006/01/26 06:40:05 UTC
svn commit: r372455 [9/11] - in /incubator/solr/trunk: ./ src/ src/apps/
src/apps/SolarTest/ src/apps/SolarTest/src/ src/java/ src/java/org/
src/java/org/apache/ src/java/org/apache/solr/
src/java/org/apache/solr/analysis/ src/java/org/apache/solr/core...
Added: incubator/solr/trunk/src/java/org/apache/solr/util/StrUtils.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/util/StrUtils.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/util/StrUtils.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/util/StrUtils.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,202 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.IOException;
+
+/**
+ * @author yonik
+ * @version $Id$
+ */
+public class StrUtils {
+
+ /**
+ * Split a string based on a separator, but don't split if it's inside
+ * a string. Assume '\' escapes the next char both inside and
+ * outside strings.
+ */
+ public static List<String> splitSmart(String s, char separator) {
+ ArrayList<String> lst = new ArrayList<String>(4);
+ int pos=0, start=0, end=s.length();
+ char inString=0;
+ while (pos < end) {
+ char ch = s.charAt(pos++);
+ if (ch=='\\') { // skip escaped chars
+ pos++;
+ } else if (inString != 0 && ch==inString) {
+ inString=0;
+ } else if (ch=='\'' || ch=='"') {
+ inString=ch;
+ } else if (ch==separator && inString==0) {
+ lst.add(s.substring(start,pos-1));
+ start=pos;
+ }
+ }
+ if (start < end) {
+ lst.add(s.substring(start,end));
+ }
+
+ /***
+ if (SolrCore.log.isLoggable(Level.FINEST)) {
+ SolrCore.log.finest("splitCommand=" + lst);
+ }
+ ***/
+
+ return lst;
+ }
+
+ /** Splits a backslash escaped string on the separator.
+ * <p>
+ * Current backslash escaping supported:
+ * <br> \n \t \r \b \f are escaped the same as a Java String
+ * <br> Other characters following a backslash are produced verbatim (\c => c)
+ *
+ * @param s the string to split
+ * @param separator the separator to split on
+ * @param decode decode backslash escaping
+ */
+ public static List<String> splitSmart(String s, String separator, boolean decode) {
+ ArrayList<String> lst = new ArrayList<String>(2);
+ StringBuilder sb = new StringBuilder();
+ int pos=0, end=s.length();
+ while (pos < end) {
+ if (s.startsWith(separator,pos)) {
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ sb=new StringBuilder();
+ }
+ pos+=separator.length();
+ continue;
+ }
+
+ char ch = s.charAt(pos++);
+ if (ch=='\\') {
+ if (!decode) sb.append(ch);
+ if (pos>=end) break; // ERROR, or let it go?
+ ch = s.charAt(pos++);
+ if (decode) {
+ switch(ch) {
+ case 'n' : ch='\n';
+ case 't' : ch='\t';
+ case 'r' : ch='\r';
+ case 'b' : ch='\b';
+ case 'f' : ch='\f';
+ }
+ }
+ }
+
+ sb.append(ch);
+ }
+
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ }
+
+ return lst;
+ }
+
+
+
+ public static List<String> splitWS(String s, boolean decode) {
+ ArrayList<String> lst = new ArrayList<String>(2);
+ StringBuilder sb = new StringBuilder();
+ int pos=0, end=s.length();
+ while (pos < end) {
+ char ch = s.charAt(pos++);
+ if (Character.isWhitespace(ch)) {
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ sb=new StringBuilder();
+ }
+ continue;
+ }
+
+ if (ch=='\\') {
+ if (!decode) sb.append(ch);
+ if (pos>=end) break; // ERROR, or let it go?
+ ch = s.charAt(pos++);
+ if (decode) {
+ switch(ch) {
+ case 'n' : ch='\n';
+ case 't' : ch='\t';
+ case 'r' : ch='\r';
+ case 'b' : ch='\b';
+ case 'f' : ch='\f';
+ }
+ }
+ }
+
+ sb.append(ch);
+ }
+
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ }
+
+ return lst;
+ }
+
+ public static List<String> toLower(List<String> strings) {
+ ArrayList<String> ret = new ArrayList<String>(strings.size());
+ for (String str : strings) {
+ ret.add(str.toLowerCase());
+ }
+ return ret;
+ }
+
+
+
+ /** Return if a string starts with '1', 't', or 'T'
+ * and return false otherwise.
+ */
+ public static boolean parseBoolean(String s) {
+ char ch = s.length()>0 ? s.charAt(0) : 0;
+ return (ch=='1' || ch=='t' || ch=='T');
+ }
+
+ /**
+ * URLEncodes a value, replacing only enough chars so that
+ * the URL may be unambiguously pasted back into a browser.
+ * <p>
+ * Characters with a numeric value less than 32 are encoded.
+ * &,=,%,+,space are encoded.
+ * <p>
+ */
+ public static void partialURLEncodeVal(Appendable dest, String val) throws IOException {
+ for (int i=0; i<val.length(); i++) {
+ char ch = val.charAt(i);
+ if (ch < 32) {
+ dest.append('%');
+ // Hmmm, if we used StringBuilder rather than Appendable, it
+ // could add an integer more efficiently.
+ dest.append(Integer.toString(ch));
+ } else {
+ switch (ch) {
+ case ' ': dest.append('+'); break;
+ case '&': dest.append("%26"); break;
+ case '%': dest.append("%25"); break;
+ case '=': dest.append("%3D"); break;
+ case '+': dest.append("%2B"); break;
+ default : dest.append(ch); break;
+ }
+ }
+ }
+ }
+
+}
Propchange: incubator/solr/trunk/src/java/org/apache/solr/util/StrUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/java/org/apache/solr/util/XML.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/util/XML.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/util/XML.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/util/XML.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,144 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.io.Writer;
+import java.io.IOException;
+
+/**
+ * @author yonik
+ * @version $Id$
+ */
+public class XML {
+
+ //
+ // copied from some of my personal code... -YCS
+ // table created from python script.
+ // only have to escape quotes in attribute values, and don't really have to escape '>'
+ // many chars less than 0x20 are *not* valid XML, even when escaped!
+ // for example, <foo>�<foo> is invalid XML.
+ private static final String[] chardata_escapes=
+ {"#0;","#1;","#2;","#3;","#4;","#5;","#6;","#7;","#8;",null,null,"#11;","#12;",null,"#14;","#15;","#16;","#17;","#18;","#19;","#20;","#21;","#22;","#23;","#24;","#25;","#26;","#27;","#28;","#29;","#30;","#31;",null,null,null,null,null,null,"&",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"<"};
+
+
+ /*****************************************
+ #Simple python script used to generate the escape table above. -YCS
+ #
+ #use individual char arrays or one big char array for better efficiency
+ # or byte array?
+ #other={'&':'amp', '<':'lt', '>':'gt', "'":'apos', '"':'quot'}
+ #
+ other={'&':'amp', '<':'lt'}
+
+ maxi=ord(max(other.keys()))+1
+ table=[None] * maxi
+ #NOTE: invalid XML chars are "escaped" as #nn; *not* &#nn; because
+ #a real XML escape would cause many strict XML parsers to choke.
+ for i in range(0x20): table[i]='#%d;' % i
+ for i in '\n\r\t ': table[ord(i)]=None
+ for k,v in other.items():
+ table[ord(k)]='&%s;' % v
+
+ result=""
+ for i in range(maxi):
+ val=table[i]
+ if not val: val='null'
+ else: val='"%s"' % val
+ result += val + ','
+
+ print result
+ ****************************************/
+
+
+/*********
+ *
+ * @param str
+ * @param out
+ * @throws IOException
+ */
+ public static void escapeCharData(String str, Writer out) throws IOException {
+ int start=0;
+ // "n" was used for counting the chars added to out...
+ // removed cause it wasn't really useful so far.
+ // int n=0;
+
+ for (int i=start; i<str.length(); i++) {
+ char ch = str.charAt(i);
+ // since I already received the char, what if I put it into
+ // a char array and wrote that to the stream instead of the
+ // string? (would cause extra GC though)
+ String subst=null;
+ if (ch<chardata_escapes.length) {
+ subst=chardata_escapes[ch];
+ }
+ if (subst != null) {
+ if (start<i) {
+ // out.write(str.substring(start,i));
+ out.write(str, start, i-start);
+ // n+=i-start;
+ }
+ out.write(subst);
+ // n+=subst.length();
+ start=i+1;
+ }
+ }
+ if (start==0) {
+ out.write(str);
+ // n += str.length();
+ } else if (start<str.length()) {
+ // out.write(str.substring(start));
+ out.write(str, start, str.length()-start);
+ // n += str.length()-start;
+ }
+ // return n;
+ }
+
+ public final static void writeXML(Writer out, String tag, String val) throws IOException {
+ out.write('<');
+ out.write(tag);
+ if (val == null) {
+ out.write("/>");
+ } else {
+ out.write('>');
+ escapeCharData(val,out);
+ out.write("</");
+ out.write(tag);
+ out.write('>');
+ }
+ }
+
+ public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ out.write('<');
+ out.write(tag);
+ for (int i=0; i<attrs.length; i++) {
+ out.write(' ');
+ out.write(attrs[i++].toString());
+ out.write("=\"");
+ out.write(attrs[i].toString());
+ out.write("\"");
+ }
+ if (val == null) {
+ out.write("/>");
+ } else {
+ out.write('>');
+ escapeCharData(val,out);
+ out.write("</");
+ out.write(tag);
+ out.write('>');
+ }
+ }
+}
Propchange: incubator/solr/trunk/src/java/org/apache/solr/util/XML.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/java/org/apache/solr/util/test/TestNumberUtils.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/java/org/apache/solr/util/test/TestNumberUtils.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/util/test/TestNumberUtils.java (added)
+++ incubator/solr/trunk/src/java/org/apache/solr/util/test/TestNumberUtils.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,491 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.test;
+
+import org.apache.solr.util.NumberUtils;
+import org.apache.solr.util.BCDUtils;
+
+import java.util.Random;
+
+/**
+ * @author yonik
+ */
+
+public class TestNumberUtils {
+
+ private static String arrstr(char[] arr, int start, int end) {
+ String str="[";
+ for (int i=start; i<end; i++) str += arr[i]+"("+(int)arr[i]+"),";
+ return str+"]";
+ }
+
+ static Random rng = new Random();
+
+ static int[] special = {0,10,100,1000,10000,Integer.MAX_VALUE, Integer.MIN_VALUE};
+ static int getSpecial() {
+ int i = rng.nextInt();
+ int j = rng.nextInt();
+ if ((i & 0x10) != 0) return j;
+ return special[(j&0x7fffffff) % special.length]* ((i & 0x20)==0?1:-1) + ((i&0x03)-1);
+ }
+
+ static long[] lspecial = {0,10,100,1000,10000,2,4,8,256,16384,32768,65536,
+ Integer.MAX_VALUE, Integer.MIN_VALUE,
+ Long.MAX_VALUE, Long.MIN_VALUE};
+ static long getLongSpecial() {
+ int i = rng.nextInt();
+ long j = rng.nextLong();
+ if ((i & 0x10) != 0) return j;
+ return lspecial[((int)j&0x7fffffff) % special.length]* ((i & 0x20)==0?1:-1) + ((i&0x03)-1);
+ }
+
+ static float[] fspecial = {0,1,2,4,8,256,16384,32768,65536,.1f,.25f
+ ,Float.NEGATIVE_INFINITY,Float.POSITIVE_INFINITY,Float.MIN_VALUE, Float.MAX_VALUE};
+ static float getFloatSpecial() {
+ int i = rng.nextInt();
+ int j = rng.nextInt();
+ float f = Float.intBitsToFloat(j);
+ if (f!=f) f=0; // get rid of NaN for comparison purposes
+ if ((i & 0x10) != 0) return f;
+ return fspecial[(j&0x7fffffff) % fspecial.length]* ((i & 0x20)==0?1:-1) + ((i&0x03)-1);
+ }
+
+ static double[] dspecial = {0,1,2,4,8,256,16384,32768,65536,.1,.25
+ ,Float.NEGATIVE_INFINITY,Float.POSITIVE_INFINITY,Float.MIN_VALUE, Float.MAX_VALUE
+ ,Double.NEGATIVE_INFINITY,Double.POSITIVE_INFINITY,Double.MIN_VALUE, Double.MAX_VALUE
+ };
+ static double getDoubleSpecial() {
+ int i = rng.nextInt();
+ long j = rng.nextLong();
+ double f = Double.longBitsToDouble(j);
+ if (f!=f) f=0; // get rid of NaN for comparison purposes
+ if ((i & 0x10) != 0) return f;
+ return dspecial[((int)j&0x7fffffff) % dspecial.length]* ((i & 0x20)==0?1:-1) + ((i&0x03)-1);
+ }
+
+
+ public static void test(Comparable n1, Comparable n2, Converter conv) {
+ String s1=n1.toString();
+ String s2=n2.toString();
+ String v1 = conv.toInternal(s1);
+ String v2 = conv.toInternal(s2);
+ String out1=conv.toExternal(v1);
+ String out2=conv.toExternal(v2);
+
+ int c1 = n1.compareTo(n2);
+ int c2 = v1.compareTo(v2);
+ if (c1==0 && !(c2==0) || c1 < 0 && !(c2<0) || c1>0 && !(c2>0)
+ || !out1.equals(s1) || !out2.equals(s2))
+ {
+ System.out.println("Comparison error:"+s1+","+s2);
+ System.out.print("v1=");
+ for (int ii=0; ii<v1.length(); ii++) {
+ System.out.print(" " + (int)v1.charAt(ii));
+ }
+ System.out.print("\nv2=");
+ for (int ii=0; ii<v2.length(); ii++) {
+ System.out.print(" " + (int)v2.charAt(ii));
+ }
+ System.out.println("\nout1='"+out1+"', out2='" + out2 + "'");
+ }
+ }
+
+
+ public static void main(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
+ int iter=1000000;
+ int arrsz=100000;
+ int ret=0;
+ int num=0;
+
+ String test="b100";
+ String clazz="NoClass";
+
+ for (int argnum=0; argnum<args.length; argnum++) {
+ String arg = args[argnum];
+ if ("-t".equals(arg)) { test=args[++argnum]; }
+ if ("-i".equals(arg)) { iter=Integer.parseInt(args[++argnum]); }
+ if ("-a".equals(arg)) { arrsz=Integer.parseInt(args[++argnum]); }
+ if ("-c".equals(arg)) { clazz=args[++argnum]; }
+ if ("-r".equals(arg)) { rng.setSeed(Long.parseLong(args[++argnum])); };
+ if ("-n".equals(arg)) { num = Integer.parseInt(args[++argnum]); };
+
+ }
+
+ // Converter conv = (Converter)(Class.forName(clazz).newInstance());
+ Class cls=null;
+ try {
+ cls = Class.forName(clazz);
+ } catch (Exception e) {
+ cls = Class.forName("solr.util.test." + clazz);
+ }
+ Converter conv = (Converter)cls.newInstance();
+
+ long startTime = System.currentTimeMillis();
+
+ if ("ispecial".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ Integer i1 = getSpecial();
+ Integer i2 = getSpecial();
+ test(i1,i2,conv);
+ }
+ }
+ else if ("lspecial".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ Long f1 = getLongSpecial();
+ Long f2 = getLongSpecial();
+ test(f1,f2,conv);
+ }
+ }
+ else if ("fspecial".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ Float f1 = getFloatSpecial();
+ Float f2 = getFloatSpecial();
+ test(f1,f2,conv);
+ }
+ }
+ else if ("dspecial".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ Double f1 = getDoubleSpecial();
+ Double f2 = getDoubleSpecial();
+ test(f1,f2,conv);
+ }
+ }
+ else if ("10kout".equals(test)) {
+ String n = Integer.toString(num);
+ char[] arr = new char[n.length()];
+ char[] arr2 = new char[n.length()+1];
+ n.getChars(0,n.length(),arr,0);
+ for (int i=0; i<iter; i++) {
+ ret += BCDUtils.base10toBase100SortableInt(arr,0,arr.length,arr2,arr2.length);
+ }
+
+ } else if ("internal".equals(test) || "external".equals(test)) {
+ int min=-1000000; int max=1000000;
+ String[] arr = new String[arrsz];
+ String[] internal = new String[arrsz];
+
+ if ("external".equals(test)) {
+ for (int i=0; i<arrsz; i++) {
+ int val = rng.nextInt();
+ // todo - move to between min and max...
+ arr[i] = Integer.toString(rng.nextInt());
+ internal[i] = conv.toInternal(arr[i]);
+ }
+ for (int i=0; i<iter; i++) {
+ int slot=i%arrsz;
+ arr[slot] = conv.toExternal(internal[slot]);
+ ret += arr[slot].length();
+ }
+ } else {
+ for (int i=0; i<arrsz; i++) {
+ int val = rng.nextInt();
+ // todo - move to between min and max...
+ arr[i] = Integer.toString(rng.nextInt());
+ }
+ for (int i=0; i<iter; i++) {
+ int slot=i%arrsz;
+ internal[slot] = conv.toInternal(arr[slot]);
+ ret += internal[slot].length();
+ }
+ }
+ } else if ("itest".equals(test) || "ltest".equals(test) || "ftest".equals(test)) {
+ long internalLen=0;
+ long externalLen=0;
+ for (int i=0; i<iter; i++) {
+ Comparable n1=null,n2=null;
+
+ if ("itest".equals(test)) {
+ Integer i1
+ = rng.nextInt();
+ Integer i2
+ = rng.nextInt();
+
+ // concentrate on small numbers for a while
+ // to try and hit boundary cases 0,1,-1,100,-100,etc
+ if (i < 10000) {
+ i1 = (i1 % 250)-125;
+ i2 = (i2 % 250)-125;
+ } else if (i < 500000) {
+ i1 = (i1 % 25000)-12500;
+ i2 = (i2 % 25000)-12500;
+ }
+
+ n1=i1;
+ n2=i2;
+ } else if ("ltest".equals(test)) {
+ Long i1 = rng.nextLong();
+ Long i2 = rng.nextLong();
+
+ // concentrate on small numbers for a while
+ // to try and hit boundary cases 0,1,-1,100,-100,etc
+ if (i < 10000) {
+ i1 = (long)(i1 % 250)-125;
+ i2 = (long)(i2 % 250)-125;
+ } else if (i < 500000) {
+ i1 = (long)(i1 % 25000)-12500;
+ i2 = (long)(i2 % 25000)-12500;
+ }
+
+ n1=i1;
+ n2=i2;
+ } else if ("ftest".equals(test)) {
+ Float i1;
+ Float i2;
+ if (i < 10000) {
+ i1 = (float)(rng.nextInt() % 250)-125;
+ i2 = (float)(rng.nextInt() % 250)-125;
+ } else if (i < 300000) {
+ i1 = (float)(rng.nextInt() % 2500)-1250;
+ i2 = (float)(rng.nextInt() % 2500)-1250;
+ } else if (i < 500000) {
+ i1 = rng.nextFloat() / rng.nextFloat();
+ i2 = rng.nextFloat() / rng.nextFloat();
+ } else {
+ i1 = Float.intBitsToFloat(rng.nextInt());
+ i2 = Float.intBitsToFloat(rng.nextInt());
+ }
+ n1=i1;
+ n2=i2;
+ }
+ String s1=n1.toString();
+ String s2=n2.toString();
+ String v1 = conv.toInternal(s1);
+ String v2 = conv.toInternal(s2);
+ String out1=conv.toExternal(v1);
+ String out2=conv.toExternal(v2);
+
+ externalLen += s1.length();
+ internalLen += v1.length();
+
+ int c1 = n1.compareTo(n2);
+ int c2 = v1.compareTo(v2);
+ if (c1==0 && !(c2==0) || c1 < 0 && !(c2<0) || c1>0 && !(c2>0)
+ || !out1.equals(s1) || !out2.equals(s2))
+ {
+ System.out.println("Comparison error:"+s1+","+s2);
+ System.out.print("v1=");
+ for (int ii=0; ii<v1.length(); ii++) {
+ System.out.print(" " + (int)v1.charAt(ii));
+ }
+ System.out.print("\nv2=");
+ for (int ii=0; ii<v2.length(); ii++) {
+ System.out.print(" " + (int)v2.charAt(ii));
+ }
+ System.out.println("\nout1='"+out1+"', out2='" + out2 + "'");
+
+ }
+ }
+ }
+
+
+ /******************
+ int sz=20;
+ char[] arr1 = new char[sz];
+ char[] arr2 = new char[sz];
+ char[] arr3 = new char[sz];
+ if ("noconv".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ int val = rng.nextInt();
+ String istr = Integer.toString(val);
+ int n = istr.length();
+ Integer.toString(val).getChars(0, n, arr1, 0);
+ String nStr = new String(arr1,0,n);
+ if (!nStr.equals(istr)) {
+ System.out.println("ERROR! input="+istr+" output="+nStr);
+ System.out.println(arrstr(arr1,0,n));
+ }
+ }
+ } else if ("b100".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ int val = rng.nextInt();
+ String istr = Integer.toString(val);
+ int n = istr.length();
+ Integer.toString(val).getChars(0, n, arr1, 0);
+
+ int b100_start = NumberUtils.base10toBase100(arr1,0,n,arr2,sz);
+ int b10_len = NumberUtils.base100toBase10(arr2,b100_start,sz,arr3,0);
+
+ String nStr = new String(arr3,0,b10_len);
+ if (!nStr.equals(istr)) {
+ System.out.println("ERROR! input="+istr+" output="+nStr);
+ System.out.println(arrstr(arr1,0,n));
+ System.out.println(arrstr(arr2,b100_start,sz));
+ System.out.println(arrstr(arr3,0,b10_len));
+ }
+
+ }
+ } else if ("b100sParse".equals(test)) {
+ int min=-1000000; int max=1000000;
+ String[] arr = new String[arrsz];
+ String[] internal = new String[arrsz];
+ for (int i=0; i<arrsz; i++) {
+ int val = rng.nextInt();
+ // todo - move to between min and max...
+ arr[i] = Integer.toString(rng.nextInt());
+ }
+ for (int i=0; i<iter; i++) {
+ int slot=i%arrsz;
+ internal[slot] = NumberUtils.base10toBase100SortableInt(arr[i%arrsz]);
+ ret += internal[slot].length();
+ }
+ } else if ("intParse".equals(test)) {
+ int min=-1000000; int max=1000000;
+ String[] arr = new String[arrsz];
+ String[] internal = new String[arrsz];
+ for (int i=0; i<arrsz; i++) {
+ int val = rng.nextInt();
+ // todo - move to between min and max...
+ arr[i] = Integer.toString(rng.nextInt());
+ }
+ for (int i=0; i<iter; i++) {
+ int slot=i%arrsz;
+ int val = Integer.parseInt(arr[i%arrsz]);
+ String sval = Integer.toString(val);
+ internal[slot] = sval;
+ ret += internal[slot].length();
+ }
+ } else if ("b100s".equals(test)) {
+ for (int i=0; i<iter; i++) {
+ Integer i1 = rng.nextInt();
+ Integer i2 = rng.nextInt();
+
+ // concentrate on small numbers for a while
+ // to try and hit boundary cases 0,1,-1,100,-100,etc
+ if (iter < 10000) {
+ i1 = (i1 % 250)-125;
+ i2 = (i2 % 250)-125;
+ } else if (iter < 500000) {
+ i1 = (i1 % 25000)-12500;
+ i2 = (i2 % 25000)-12500;
+ }
+
+ String s1=Integer.toString(i1);
+ String s2=Integer.toString(i2);
+ String v1 = NumberUtils.base10toBase10kSortableInt(s1);
+ String v2 = NumberUtils.base10toBase10kSortableInt(s2);
+ String out1=NumberUtils.base10kSortableIntToBase10(v1);
+ String out2=NumberUtils.base10kSortableIntToBase10(v2);
+
+ int c1 = i1.compareTo(i2);
+ int c2 = v1.compareTo(v2);
+ if (c1==0 && c2 !=0 || c1 < 0 && c2 >= 0 || c1 > 0 && c2 <=0
+ || !out1.equals(s1) || !out2.equals(s2))
+ {
+ System.out.println("Comparison error:"+s1+","+s2);
+ System.out.print("v1=");
+ for (int ii=0; ii<v1.length(); ii++) {
+ System.out.print(" " + (int)v1.charAt(ii));
+ }
+ System.out.print("\nv2=");
+ for (int ii=0; ii<v2.length(); ii++) {
+ System.out.print(" " + (int)v2.charAt(ii));
+ }
+ System.out.println("\nout1='"+out1+"', out2='" + out2 + "'");
+
+ }
+
+
+
+
+
+ }
+ }
+ ****/
+
+ long endTime = System.currentTimeMillis();
+ System.out.println("time="+(endTime-startTime));
+ System.out.println("ret="+ret);
+ }
+}
+
+
+interface Converter {
+ String toInternal(String val);
+ String toExternal(String val);
+}
+
+class Int2Int implements Converter {
+ public String toInternal(String val) {
+ return Integer.toString(Integer.parseInt(val));
+ }
+ public String toExternal(String val) {
+ return Integer.toString(Integer.parseInt(val));
+ }
+}
+
+class SortInt implements Converter {
+ public String toInternal(String val) {
+ return NumberUtils.int2sortableStr(val);
+ }
+ public String toExternal(String val) {
+ return NumberUtils.SortableStr2int(val);
+ }
+}
+
+class SortLong implements Converter {
+ public String toInternal(String val) {
+ return NumberUtils.long2sortableStr(val);
+ }
+ public String toExternal(String val) {
+ return NumberUtils.SortableStr2long(val);
+ }
+}
+
+class Float2Float implements Converter {
+ public String toInternal(String val) {
+ return Float.toString(Float.parseFloat(val));
+ }
+ public String toExternal(String val) {
+ return Float.toString(Float.parseFloat(val));
+ }
+}
+
+class SortFloat implements Converter {
+ public String toInternal(String val) {
+ return NumberUtils.float2sortableStr(val);
+ }
+ public String toExternal(String val) {
+ return NumberUtils.SortableStr2floatStr(val);
+ }
+}
+
+class SortDouble implements Converter {
+ public String toInternal(String val) {
+ return NumberUtils.double2sortableStr(val);
+ }
+ public String toExternal(String val) {
+ return NumberUtils.SortableStr2doubleStr(val);
+ }
+}
+
+class Base100S implements Converter {
+ public String toInternal(String val) {
+ return BCDUtils.base10toBase100SortableInt(val);
+ }
+ public String toExternal(String val) {
+ return BCDUtils.base100SortableIntToBase10(val);
+ }
+}
+
+class Base10kS implements Converter {
+ public String toInternal(String val) {
+ return BCDUtils.base10toBase10kSortableInt(val);
+ }
+ public String toExternal(String val) {
+ return BCDUtils.base10kSortableIntToBase10(val);
+ }
+}
Propchange: incubator/solr/trunk/src/java/org/apache/solr/util/test/TestNumberUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymFilter.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymFilter.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymFilter.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymFilter.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,204 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import java.util.*;
+
+/** SynonymFilter handles multi-token synonyms with variable position increment offsets.
+ * <p>
+ * The matched tokens from the input stream may be optionally passed through (includeOrig=true)
+ * or discarded. If the original tokens are included, the position increments may be modified
+ * to retain absolute positions after merging with the synonym tokenstream.
+ * <p>
+ * Generated synonyms will start at the same position as the first matched source token.
+ *
+ * @author yonik
+ * @version $Id: SynonymFilter.java,v 1.3 2005/12/13 05:14:52 yonik Exp $
+ */
+public class SynonymFilter extends TokenFilter {
+
+ private final SynonymMap map; // Map<String, SynonymMap>
+ private final boolean ignoreCase;
+ private Iterator replacement; // iterator over generated tokens
+
+ public SynonymFilter(TokenStream in, SynonymMap map, boolean ignoreCase) {
+ super(in);
+ this.map = map;
+ this.ignoreCase = ignoreCase;
+ }
+
+
+ /*
+ * Need to worry about multiple scenarios:
+ * - need to go for the longest match
+ * a b => foo #shouldn't match if "a b" is followed by "c d"
+ * a b c d => bar
+ * - need to backtrack - retry matches for tokens already read
+ * a b c d => foo
+ * b c => bar
+ * If the input stream is "a b c x", one will consume "a b c d"
+ * trying to match the first rule... all but "a" should be
+ * pushed back so a match may be made on "b c".
+ * - don't try and match generated tokens (thus need separate queue)
+ * matching is not recursive.
+ * - handle optional generation of original tokens in all these cases,
+ * merging token streams to preserve token positions.
+ * - preserve original positionIncrement of first matched token
+ */
+
+
+ public Token next() throws IOException {
+ while (true) {
+ // if there are any generated tokens, return them... don't try any
+ // matches against them, as we specifically don't want recursion.
+ if (replacement!=null && replacement.hasNext()) {
+ return (Token)replacement.next();
+ }
+
+ // common case fast-path of first token not matching anything
+ Token firstTok = nextTok();
+ if (firstTok ==null) return null;
+ String str = ignoreCase ? firstTok.termText.toLowerCase() : firstTok.termText;
+ Object o = map.submap!=null ? map.submap.get(str) : null;
+ if (o == null) return firstTok;
+
+ // OK, we matched a token, so find the longest match.
+
+ // since matched is only used for matches >= 2, defer creation until now
+ if (matched==null) matched=new LinkedList();
+
+ SynonymMap result = match((SynonymMap)o);
+
+ if (result==null) {
+ // no match, simply return the first token read.
+ return firstTok;
+ }
+
+ // reuse, or create new one each time?
+ ArrayList generated = new ArrayList(result.synonyms.length + matched.size() + 1);
+
+ //
+ // there was a match... let's generate the new tokens, merging
+ // in the matched tokens (position increments need adjusting)
+ //
+ Token lastTok = matched.isEmpty() ? firstTok : (Token)matched.getLast();
+ boolean includeOrig = result.includeOrig();
+
+ Token origTok = includeOrig ? firstTok : null;
+ int origPos = firstTok.getPositionIncrement(); // position of origTok in the original stream
+ int repPos=0; // curr position in replacement token stream
+ int pos=0; // current position in merged token stream
+
+ for (int i=0; i<result.synonyms.length; i++) {
+ Token repTok = result.synonyms[i];
+ Token newTok = new Token(repTok.termText, firstTok.startOffset, lastTok.endOffset, firstTok.type);
+ repPos += repTok.getPositionIncrement();
+ if (i==0) repPos=origPos; // make position of first token equal to original
+
+ // if necessary, insert original tokens and adjust position increment
+ while (origTok != null && origPos <= repPos) {
+ origTok.setPositionIncrement(origPos-pos);
+ generated.add(origTok);
+ pos += origTok.getPositionIncrement();
+ origTok = matched.isEmpty() ? null : (Token)matched.removeFirst();
+ if (origTok != null) origPos += origTok.getPositionIncrement();
+ }
+
+ newTok.setPositionIncrement(repPos - pos);
+ generated.add(newTok);
+ pos += newTok.getPositionIncrement();
+ }
+
+ // finish up any leftover original tokens
+ while (origTok!=null) {
+ origTok.setPositionIncrement(origPos-pos);
+ generated.add(origTok);
+ pos += origTok.getPositionIncrement();
+ origTok = matched.isEmpty() ? null : (Token)matched.removeFirst();
+ if (origTok != null) origPos += origTok.getPositionIncrement();
+ }
+
+ // what if we replaced a longer sequence with a shorter one?
+ // a/0 b/5 => foo/0
+ // should I re-create the gap on the next buffered token?
+
+ replacement = generated.iterator();
+ // Now return to the top of the loop to read and return the first
+ // generated token.. The reason this is done is that we may have generated
+ // nothing at all, and may need to continue with more matching logic.
+ }
+ }
+
+
+ //
+ // Defer creation of the buffer until the first time it is used to
+ // optimize short fields with no matches.
+ //
+ private LinkedList buffer;
+ private LinkedList matched;
+
+ // TODO: use ArrayList for better performance?
+
+ private Token nextTok() throws IOException {
+ if (buffer!=null && !buffer.isEmpty()) {
+ return (Token)buffer.removeFirst();
+ } else {
+ return input.next();
+ }
+ }
+
+ private void pushTok(Token t) {
+ if (buffer==null) buffer=new LinkedList();
+ buffer.addFirst(t);
+ }
+
+
+
+ private SynonymMap match(SynonymMap map) throws IOException {
+ SynonymMap result = null;
+
+ if (map.submap != null) {
+ Token tok = nextTok();
+ if (tok != null) {
+ // check for positionIncrement!=1? if>1, should not match, if==0, check multiple at this level?
+ String str = ignoreCase ? tok.termText.toLowerCase() : tok.termText;
+
+ SynonymMap subMap = (SynonymMap)map.submap.get(str);
+
+ if (subMap !=null) {
+ // recurse
+ result = match(subMap);
+ }
+ if (result != null) {
+ matched.addFirst(tok);
+ } else {
+ // push back unmatched token
+ pushTok(tok);
+ }
+ }
+ }
+
+ // if no longer sequence matched, so if this node has synonyms, it's the match.
+ if (result==null && map.synonyms!=null) {
+ result = map;
+ }
+
+ return result;
+ }
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymMap.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymMap.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymMap.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymMap.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,143 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.util.*;
+
+/** Mapping rules for use with {@link SynonymFilter}
+ *
+ * @author yonik
+ * @version $Id: SynonymMap.java,v 1.2 2005/12/13 05:15:08 yonik Exp $
+ */
+public class SynonymMap {
+ Map submap; // recursive: Map<String, SynonymMap>
+ Token[] synonyms;
+ int flags;
+
+ static final int INCLUDE_ORIG=0x01;
+
+ public boolean includeOrig() { return (flags & INCLUDE_ORIG) != 0; }
+
+ /**
+ * @param singleMatch List<String>, the sequence of strings to match
+ * @param replacement List<Token> the list of tokens to use on a match
+ * @param includeOrig sets a flag on this mapping signaling the generation of matched tokens in addition to the replacement tokens
+ * @param mergeExisting merge the replacement tokens with any other mappings that exist
+ */
+ public void add(List singleMatch, List replacement, boolean includeOrig, boolean mergeExisting) {
+ SynonymMap currMap = this;
+ for (Iterator iter = singleMatch.iterator(); iter.hasNext();) {
+ String str = (String)iter.next();
+ if (currMap.submap==null) {
+ currMap.submap = new HashMap(1);
+ }
+
+ SynonymMap map = (SynonymMap)currMap.submap.get(str);
+ if (map==null) {
+ map = new SynonymMap();
+ currMap.submap.put(str, map);
+ }
+
+ currMap = map;
+ }
+
+ if (currMap.synonyms != null && !mergeExisting) {
+ throw new RuntimeException("SynonymFilter: there is already a mapping for " + singleMatch);
+ }
+ List superset = currMap.synonyms==null ? replacement :
+ mergeTokens(Arrays.asList(currMap.synonyms), replacement);
+ currMap.synonyms = (Token[])superset.toArray(new Token[superset.size()]);
+ if (includeOrig) currMap.flags |= INCLUDE_ORIG;
+ }
+
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer("<");
+ if (synonyms!=null) {
+ sb.append("[");
+ for (int i=0; i<synonyms.length; i++) {
+ if (i!=0) sb.append(',');
+ sb.append(synonyms[i]);
+ }
+ if ((flags & INCLUDE_ORIG)!=0) {
+ sb.append(",ORIG");
+ }
+ sb.append("],");
+ }
+ sb.append(submap);
+ sb.append(">");
+ return sb.toString();
+ }
+
+
+
+ /** Produces a List<Token> from a List<String> */
+ public static List makeTokens(List strings) {
+ List ret = new ArrayList(strings.size());
+ for (Iterator iter = strings.iterator(); iter.hasNext();) {
+ Token newTok = new Token((String)iter.next(),0,0,"SYNONYM");
+ ret.add(newTok);
+ }
+ return ret;
+ }
+
+
+ /**
+ * Merge two lists of tokens, producing a single list with manipulated positionIncrements so that
+ * the tokens end up at the same position.
+ *
+ * Example: [a b] merged with [c d] produces [a/b c/d] ('/' denotes tokens in the same position)
+ * Example: [a,5 b,2] merged with [c d,4 e,4] produces [c a,5/d b,2 e,2] (a,n means a has posInc=n)
+ *
+ */
+ public static List mergeTokens(List lst1, List lst2) {
+ ArrayList result = new ArrayList();
+ if (lst1 ==null || lst2 ==null) {
+ if (lst2 != null) result.addAll(lst2);
+ if (lst1 != null) result.addAll(lst1);
+ return result;
+ }
+
+ int pos=0;
+ Iterator iter1=lst1.iterator();
+ Iterator iter2=lst2.iterator();
+ Token tok1 = iter1.hasNext() ? (Token)iter1.next() : null;
+ Token tok2 = iter2.hasNext() ? (Token)iter2.next() : null;
+ int pos1 = tok1!=null ? tok1.getPositionIncrement() : 0;
+ int pos2 = tok2!=null ? tok2.getPositionIncrement() : 0;
+ while(tok1!=null || tok2!=null) {
+ while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
+ Token tok = new Token(tok1.termText, tok1.startOffset, tok1.endOffset, tok1.type);
+ tok.setPositionIncrement(pos1-pos);
+ result.add(tok);
+ pos=pos1;
+ tok1 = iter1.hasNext() ? (Token)iter1.next() : null;
+ pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
+ }
+ while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
+ Token tok = new Token(tok2.termText, tok2.startOffset, tok2.endOffset, tok2.type);
+ tok.setPositionIncrement(pos2-pos);
+ result.add(tok);
+ pos=pos2;
+ tok2 = iter2.hasNext() ? (Token)iter2.next() : null;
+ pos2 += tok2!=null ? tok2.getPositionIncrement() : 0;
+ }
+ }
+ return result;
+ }
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/SynonymMap.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/TestSynonymFilter.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/TestSynonymFilter.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/TestSynonymFilter.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/TestSynonymFilter.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,279 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import junit.framework.TestCase;
+
+import java.util.*;
+import java.io.IOException;
+
+/**
+ * @author yonik
+ * @version $Id: TestSynonymFilter.java,v 1.2 2005/12/12 18:07:23 yonik Exp $
+ */
+public class TestSynonymFilter extends TestCase {
+
+ public List strings(String str) {
+ String[] arr = str.split(" ");
+ return Arrays.asList(arr);
+ }
+
+ /***
+ * Return a list of tokens according to a test string format:
+ * a b c => returns List<Token> [a,b,c]
+ * a/b => tokens a and b share the same spot (b.positionIncrement=0)
+ * a,3/b/c => a,b,c all share same position (a.positionIncrement=3, b.positionIncrement=0, c.positionIncrement=0)
+ */
+ public List tokens(String str) {
+ String[] arr = str.split(" ");
+ List result = new ArrayList();
+ for (int i=0; i<arr.length; i++) {
+ String[] toks = arr[i].split("/");
+ String[] params = toks[0].split(",");
+ Token t = new Token(params[0],0,0,"TEST");
+ if (params.length > 1) t.setPositionIncrement(Integer.parseInt(params[1]));
+ result.add(t);
+ for (int j=1; j<toks.length; j++) {
+ t = new Token(toks[j],0,0,"TEST");
+ t.setPositionIncrement(0);
+ result.add(t);
+ }
+ }
+ return result;
+ }
+
+ public List getTokList(SynonymMap dict, String input, boolean includeOrig) throws IOException {
+ ArrayList lst = new ArrayList();
+ final List toks = tokens(input);
+ TokenStream ts = new TokenStream() {
+ Iterator iter = toks.iterator();
+ public Token next() throws IOException {
+ return iter.hasNext() ? (Token)iter.next() : null;
+ }
+ };
+
+ SynonymFilter sf = new SynonymFilter(ts, dict, true);
+
+ while(true) {
+ Token t = sf.next();
+ if (t==null) return lst;
+ lst.add(t);
+ }
+ }
+
+ public List tok2str(List tokLst) {
+ ArrayList lst = new ArrayList();
+ for (Iterator iter = tokLst.iterator(); iter.hasNext();) {
+ lst.add(((Token)(iter.next())).termText());
+ }
+ return lst;
+ }
+
+
+ public void assertTokEqual(List a, List b) {
+ assertTokEq(a,b);
+ assertTokEq(b,a);
+ }
+
+ private void assertTokEq(List a, List b) {
+ int pos=0;
+ for (Iterator iter = a.iterator(); iter.hasNext();) {
+ Token tok = (Token)iter.next();
+ pos += tok.getPositionIncrement();
+ if (!tokAt(b, tok.termText(), pos)) {
+ fail(a + "!=" + b);
+ }
+ }
+ }
+
+ public boolean tokAt(List lst, String val, int tokPos) {
+ int pos=0;
+ for (Iterator iter = lst.iterator(); iter.hasNext();) {
+ Token tok = (Token)iter.next();
+ pos += tok.getPositionIncrement();
+ if (pos==tokPos && tok.termText().equals(val)) return true;
+ }
+ return false;
+ }
+
+
+ public void testMatching() throws IOException {
+ SynonymMap map = new SynonymMap();
+
+ boolean orig = false;
+ boolean merge = true;
+ map.add(strings("a b"), tokens("ab"), orig, merge);
+ map.add(strings("a c"), tokens("ac"), orig, merge);
+ map.add(strings("a"), tokens("aa"), orig, merge);
+ map.add(strings("b"), tokens("bb"), orig, merge);
+ map.add(strings("z x c v"), tokens("zxcv"), orig, merge);
+ map.add(strings("x c"), tokens("xc"), orig, merge);
+
+ // System.out.println(map);
+ // System.out.println(getTokList(map,"a",false));
+
+ assertTokEqual(getTokList(map,"$",false), tokens("$"));
+ assertTokEqual(getTokList(map,"a",false), tokens("aa"));
+ assertTokEqual(getTokList(map,"a $",false), tokens("aa $"));
+ assertTokEqual(getTokList(map,"$ a",false), tokens("$ aa"));
+ assertTokEqual(getTokList(map,"a a",false), tokens("aa aa"));
+ assertTokEqual(getTokList(map,"b",false), tokens("bb"));
+ assertTokEqual(getTokList(map,"z x c v",false), tokens("zxcv"));
+ assertTokEqual(getTokList(map,"z x c $",false), tokens("z xc $"));
+
+ // repeats
+ map.add(strings("a b"), tokens("ab"), orig, merge);
+ map.add(strings("a b"), tokens("ab"), orig, merge);
+ assertTokEqual(getTokList(map,"a b",false), tokens("ab"));
+
+ // check for lack of recursion
+ map.add(strings("zoo"), tokens("zoo"), orig, merge);
+ assertTokEqual(getTokList(map,"zoo zoo $ zoo",false), tokens("zoo zoo $ zoo"));
+ map.add(strings("zoo"), tokens("zoo zoo"), orig, merge);
+ assertTokEqual(getTokList(map,"zoo zoo $ zoo",false), tokens("zoo zoo zoo zoo $ zoo zoo"));
+ }
+
+ public void testIncludeOrig() throws IOException {
+ SynonymMap map = new SynonymMap();
+
+ boolean orig = true;
+ boolean merge = true;
+ map.add(strings("a b"), tokens("ab"), orig, merge);
+ map.add(strings("a c"), tokens("ac"), orig, merge);
+ map.add(strings("a"), tokens("aa"), orig, merge);
+ map.add(strings("b"), tokens("bb"), orig, merge);
+ map.add(strings("z x c v"), tokens("zxcv"), orig, merge);
+ map.add(strings("x c"), tokens("xc"), orig, merge);
+
+ // System.out.println(map);
+ // System.out.println(getTokList(map,"a",false));
+
+ assertTokEqual(getTokList(map,"$",false), tokens("$"));
+ assertTokEqual(getTokList(map,"a",false), tokens("a/aa"));
+ assertTokEqual(getTokList(map,"a",false), tokens("a/aa"));
+ assertTokEqual(getTokList(map,"$ a",false), tokens("$ a/aa"));
+ assertTokEqual(getTokList(map,"a $",false), tokens("a/aa $"));
+ assertTokEqual(getTokList(map,"$ a !",false), tokens("$ a/aa !"));
+ assertTokEqual(getTokList(map,"a a",false), tokens("a/aa a/aa"));
+ assertTokEqual(getTokList(map,"b",false), tokens("b/bb"));
+ assertTokEqual(getTokList(map,"z x c v",false), tokens("z/zxcv x c v"));
+ assertTokEqual(getTokList(map,"z x c $",false), tokens("z x/xc c $"));
+
+ // check for lack of recursion
+ map.add(strings("zoo zoo"), tokens("zoo"), orig, merge);
+ assertTokEqual(getTokList(map,"zoo zoo $ zoo",false), tokens("zoo/zoo zoo/zoo $ zoo/zoo"));
+ map.add(strings("zoo"), tokens("zoo zoo"), orig, merge);
+ assertTokEqual(getTokList(map,"zoo zoo $ zoo",false), tokens("zoo/zoo zoo $ zoo/zoo zoo"));
+ }
+
+
+ public void testMapMerge() throws IOException {
+ SynonymMap map = new SynonymMap();
+
+ boolean orig = false;
+ boolean merge = true;
+ map.add(strings("a"), tokens("a5,5"), orig, merge);
+ map.add(strings("a"), tokens("a3,3"), orig, merge);
+ // System.out.println(map);
+ assertTokEqual(getTokList(map,"a",false), tokens("a3 a5,2"));
+
+ map.add(strings("b"), tokens("b3,3"), orig, merge);
+ map.add(strings("b"), tokens("b5,5"), orig, merge);
+ //System.out.println(map);
+ assertTokEqual(getTokList(map,"b",false), tokens("b3 b5,2"));
+
+
+ map.add(strings("a"), tokens("A3,3"), orig, merge);
+ map.add(strings("a"), tokens("A5,5"), orig, merge);
+ assertTokEqual(getTokList(map,"a",false), tokens("a3/A3 a5,2/A5"));
+
+ map.add(strings("a"), tokens("a1"), orig, merge);
+ assertTokEqual(getTokList(map,"a",false), tokens("a1 a3,2/A3 a5,2/A5"));
+
+ map.add(strings("a"), tokens("a2,2"), orig, merge);
+ map.add(strings("a"), tokens("a4,4 a6,2"), orig, merge);
+ assertTokEqual(getTokList(map,"a",false), tokens("a1 a2 a3/A3 a4 a5/A5 a6"));
+ }
+
+
+ public void testOverlap() throws IOException {
+ SynonymMap map = new SynonymMap();
+
+ boolean orig = false;
+ boolean merge = true;
+ map.add(strings("qwe"), tokens("qq/ww/ee"), orig, merge);
+ map.add(strings("qwe"), tokens("xx"), orig, merge);
+ map.add(strings("qwe"), tokens("yy"), orig, merge);
+ map.add(strings("qwe"), tokens("zz"), orig, merge);
+ assertTokEqual(getTokList(map,"$",false), tokens("$"));
+ assertTokEqual(getTokList(map,"qwe",false), tokens("qq/ww/ee/xx/yy/zz"));
+
+ // test merging within the map
+
+ map.add(strings("a"), tokens("a5,5 a8,3 a10,2"), orig, merge);
+ map.add(strings("a"), tokens("a3,3 a7,4 a9,2 a11,2 a111,100"), orig, merge);
+ assertTokEqual(getTokList(map,"a",false), tokens("a3 a5,2 a7,2 a8 a9 a10 a11 a111,100"));
+ }
+
+ public void testOffsets() throws IOException {
+ SynonymMap map = new SynonymMap();
+
+ boolean orig = false;
+ boolean merge = true;
+
+ // test that generated tokens start at the same offset as the original
+ map.add(strings("a"), tokens("aa"), orig, merge);
+ assertTokEqual(getTokList(map,"a,5",false), tokens("aa,5"));
+ assertTokEqual(getTokList(map,"a,0",false), tokens("aa,0"));
+
+ // test that offset of first replacement is ignored (always takes the orig offset)
+ map.add(strings("b"), tokens("bb,100"), orig, merge);
+ assertTokEqual(getTokList(map,"b,5",false), tokens("bb,5"));
+ assertTokEqual(getTokList(map,"b,0",false), tokens("bb,0"));
+
+ // test that subsequent tokens are adjusted accordingly
+ map.add(strings("c"), tokens("cc,100 c2,2"), orig, merge);
+ assertTokEqual(getTokList(map,"c,5",false), tokens("cc,5 c2,2"));
+ assertTokEqual(getTokList(map,"c,0",false), tokens("cc,0 c2,2"));
+
+ }
+
+
+ public void testOffsetsWithOrig() throws IOException {
+ SynonymMap map = new SynonymMap();
+
+ boolean orig = true;
+ boolean merge = true;
+
+ // test that generated tokens start at the same offset as the original
+ map.add(strings("a"), tokens("aa"), orig, merge);
+ assertTokEqual(getTokList(map,"a,5",false), tokens("a,5/aa"));
+ assertTokEqual(getTokList(map,"a,0",false), tokens("a,0/aa"));
+
+ // test that offset of first replacement is ignored (always takes the orig offset)
+ map.add(strings("b"), tokens("bb,100"), orig, merge);
+ assertTokEqual(getTokList(map,"b,5",false), tokens("bb,5/b"));
+ assertTokEqual(getTokList(map,"b,0",false), tokens("bb,0/b"));
+
+ // test that subsequent tokens are adjusted accordingly
+ map.add(strings("c"), tokens("cc,100 c2,2"), orig, merge);
+ assertTokEqual(getTokList(map,"c,5",false), tokens("cc,5/c c2,2"));
+ assertTokEqual(getTokList(map,"c,0",false), tokens("cc,0/c c2,2"));
+ }
+
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/analysis/TestSynonymFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/ConstantScorePrefixQuery.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/ConstantScorePrefixQuery.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/ConstantScorePrefixQuery.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/ConstantScorePrefixQuery.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+
+import java.io.IOException;
+
+/**
+ * @author yonik
+ * @version $Id: ConstantScorePrefixQuery.java,v 1.2 2005/09/15 14:32:41 yonik Exp $
+ */
+public class ConstantScorePrefixQuery extends Query {
+ private final Term prefix;
+
+ public ConstantScorePrefixQuery(Term prefix) {
+ this.prefix = prefix;
+ }
+
+ /** Returns the prefix for this query */
+ public Term getPrefix() { return prefix; }
+
+ public Query rewrite(IndexReader reader) throws IOException {
+ // TODO: if number of terms are low enough, rewrite to a BooleanQuery
+ // for potentially faster execution.
+ // TODO: cache the bitset somewhere instead of regenerating it
+ Query q = new ConstantScoreQuery(new PrefixFilter(prefix));
+ q.setBoost(getBoost());
+ return q;
+ }
+
+ /** Prints a user-readable version of this query. */
+ public String toString(String field)
+ {
+ StringBuffer buffer = new StringBuffer();
+ if (!prefix.field().equals(field)) {
+ buffer.append(prefix.field());
+ buffer.append(":");
+ }
+ buffer.append(prefix.text());
+ buffer.append('*');
+ if (getBoost() != 1.0f) {
+ buffer.append("^");
+ buffer.append(Float.toString(getBoost()));
+ }
+ return buffer.toString();
+ }
+
+ /** Returns true if <code>o</code> is equal to this. */
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ConstantScorePrefixQuery)) return false;
+ ConstantScorePrefixQuery other = (ConstantScorePrefixQuery) o;
+ return this.prefix.equals(other.prefix) && this.getBoost()==other.getBoost();
+ }
+
+ /** Returns a hash code value for this object.*/
+ public int hashCode() {
+ int h = prefix.hashCode() ^ Float.floatToIntBits(getBoost());
+ h ^= (h << 14) | (h >>> 19); // reversible (1 to 1) transformation unique to ConstantScorePrefixQuery
+ return h;
+ }
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/ConstantScorePrefixQuery.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PrefixFilter.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PrefixFilter.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PrefixFilter.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PrefixFilter.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,95 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+
+import java.util.BitSet;
+import java.io.IOException;
+
+/**
+ * @author yonik
+ * @version $Id: PrefixFilter.java,v 1.1 2005/06/10 05:47:32 yonik Exp $
+ */
+public class PrefixFilter extends Filter {
+ protected final Term prefix;
+
+ PrefixFilter(Term prefix) {
+ this.prefix = prefix;
+ }
+
+ Term getPrefix() { return prefix; }
+
+ public BitSet bits(IndexReader reader) throws IOException {
+ final BitSet bitSet = new BitSet(reader.maxDoc());
+ new PrefixGenerator(prefix) {
+ public void handleDoc(int doc) {
+ bitSet.set(doc);
+ }
+ }.generate(reader);
+ return bitSet;
+ }
+}
+
+
+// keep this protected until I decide if it's a good way
+// to separate id generation from collection (or should
+// I just reuse hitcollector???)
+interface IdGenerator {
+ public void generate(IndexReader reader) throws IOException;
+ public void handleDoc(int doc);
+}
+
+
+abstract class PrefixGenerator implements IdGenerator {
+ protected final Term prefix;
+
+ PrefixGenerator(Term prefix) {
+ this.prefix = prefix;
+ }
+
+ public void generate(IndexReader reader) throws IOException {
+ TermEnum enumerator = reader.terms(prefix);
+ TermDocs termDocs = reader.termDocs();
+
+ try {
+
+ String prefixText = prefix.text();
+ String prefixField = prefix.field();
+ do {
+ Term term = enumerator.term();
+ if (term != null &&
+ term.text().startsWith(prefixText) &&
+ term.field() == prefixField)
+ {
+ termDocs.seek(term);
+ while (termDocs.next()) {
+ handleDoc(termDocs.doc());
+ }
+ } else {
+ break;
+ }
+ } while (enumerator.next());
+ } finally {
+ termDocs.close();
+ enumerator.close();
+ }
+ }
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PrefixFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PublicFieldSortedHitQueue.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PublicFieldSortedHitQueue.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PublicFieldSortedHitQueue.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PublicFieldSortedHitQueue.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+/**
+ * FieldSortedHitQueue that is public (can be created and accessed from other packages)
+ *
+ * @author yonik
+ * @version $Id: PublicFieldSortedHitQueue.java,v 1.3 2005/11/11 21:57:56 yonik Exp $
+ */
+
+import org.apache.lucene.index.IndexReader;
+import java.io.IOException;
+
+public class PublicFieldSortedHitQueue extends FieldSortedHitQueue {
+ public PublicFieldSortedHitQueue (IndexReader reader, SortField[] fields, int size) throws IOException {
+ super(reader, fields, size);
+ }
+
+ int totalHits;
+ public int getTotalHits() { return totalHits; }
+
+ public boolean insert(FieldDoc element) {
+ totalHits++;
+ return super.insert(element);
+ }
+}
\ No newline at end of file
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/PublicFieldSortedHitQueue.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/DocValues.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/DocValues.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/DocValues.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/DocValues.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,47 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Represents field values as different types.
+ * Normally created via a {@link ValueSource} for a particular field and reader.
+ * <br>
+ * Often used by {@link FunctionFactory} implementations.
+ *
+ * @author yonik
+ * @version $Id: DocValues.java,v 1.1 2005/11/22 05:23:20 yonik Exp $
+ */
+
+// DocValues is distinct from ValueSource because
+// there needs to be an object created at query evaluation time that
+// is not referenced by the query itself because:
+// - Query objects should be MT safe
+// - For caching, Query objects are often used as keys... you don't
+// want the Query carrying around big objects
+public abstract class DocValues {
+ public float floatVal(int doc) { throw new UnsupportedOperationException(); }
+ public int intVal(int doc) { throw new UnsupportedOperationException(); }
+ public long longVal(int doc) { throw new UnsupportedOperationException(); }
+ public double doubleVal(int doc) { throw new UnsupportedOperationException(); }
+ public String strVal(int doc) { throw new UnsupportedOperationException(); }
+ public abstract String toString(int doc);
+ public Explanation explain(int doc) {
+ return new Explanation(floatVal(doc), toString(doc));
+ }
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/DocValues.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FieldCacheSource.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FieldCacheSource.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FieldCacheSource.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FieldCacheSource.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,59 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.search.FieldCache;
+
+/**
+ * A base class for ValueSource implementations that retrieve values for
+ * a single field from the {@link org.apache.lucene.search.FieldCache}.
+ *
+ * @author yonik
+ * @version $Id: FieldCacheSource.java,v 1.1 2005/11/22 05:23:20 yonik Exp $
+ */
+public abstract class FieldCacheSource extends ValueSource {
+ protected String field;
+ protected FieldCache cache = FieldCache.DEFAULT;
+
+ public FieldCacheSource(String field) {
+ this.field=field;
+ }
+
+ public void setFieldCache(FieldCache cache) {
+ this.cache = cache;
+ }
+
+ public FieldCache getFieldCache() {
+ return cache;
+ }
+
+ public String description() {
+ return field;
+ }
+
+ public boolean equals(Object o) {
+ if (!(o instanceof FieldCacheSource)) return false;
+ FieldCacheSource other = (FieldCacheSource)o;
+ return this.field.equals(other.field)
+ && this.cache == other.cache;
+ }
+
+ public int hashCode() {
+ return cache.hashCode() + field.hashCode();
+ };
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FieldCacheSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FloatFieldSource.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FloatFieldSource.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FloatFieldSource.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FloatFieldSource.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.function.DocValues;
+import org.apache.lucene.search.function.ValueSource;
+import org.apache.lucene.search.FieldCache;
+
+import java.io.IOException;
+
+/**
+ * Obtains float field values from the {@link org.apache.lucene.search.FieldCache}
+ * using <code>getFloats()</code>
+ * and makes those values available as other numeric types, casting as needed.
+ *
+ * @author yonik
+ * @version $Id: FloatFieldSource.java,v 1.2 2005/11/22 05:23:20 yonik Exp $
+ */
+
+public class FloatFieldSource extends FieldCacheSource {
+ protected FieldCache.FloatParser parser;
+
+ public FloatFieldSource(String field) {
+ this(field, null);
+ }
+
+ public FloatFieldSource(String field, FieldCache.FloatParser parser) {
+ super(field);
+ this.parser = parser;
+ }
+
+ public String description() {
+ return "float(" + field + ')';
+ }
+
+ public DocValues getValues(IndexReader reader) throws IOException {
+ final float[] arr = (parser==null) ?
+ cache.getFloats(reader, field) :
+ cache.getFloats(reader, field, parser);
+ return new DocValues() {
+ public float floatVal(int doc) {
+ return arr[doc];
+ }
+
+ public int intVal(int doc) {
+ return (int)arr[doc];
+ }
+
+ public long longVal(int doc) {
+ return (long)arr[doc];
+ }
+
+ public double doubleVal(int doc) {
+ return (double)arr[doc];
+ }
+
+ public String strVal(int doc) {
+ return Float.toString(arr[doc]);
+ }
+
+ public String toString(int doc) {
+ return description() + '=' + floatVal(doc);
+ }
+ };
+ }
+
+ public boolean equals(Object o) {
+ if (o.getClass() != FloatFieldSource.class) return false;
+ FloatFieldSource other = (FloatFieldSource)o;
+ return super.equals(other)
+ && this.parser==null ? other.parser==null :
+ this.parser.getClass() == other.parser.getClass();
+ }
+
+ public int hashCode() {
+ int h = parser==null ? Float.class.hashCode() : parser.getClass().hashCode();
+ h += super.hashCode();
+ return h;
+ };
+
+}
\ No newline at end of file
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FloatFieldSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FunctionQuery.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FunctionQuery.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FunctionQuery.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FunctionQuery.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,173 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.*;
+import java.io.IOException;
+
+
+/**
+ * Returns a score for each document based on a ValueSource,
+ * often some function of the value of a field.
+ *
+ * @author yonik
+ * @version $Id: FunctionQuery.java,v 1.4 2005/11/23 04:22:20 yonik Exp $
+ */
+public class FunctionQuery extends Query {
+ ValueSource func;
+
+ /**
+ *
+ * @param func defines the function to be used for scoring
+ */
+ public FunctionQuery(ValueSource func) {
+ this.func=func;
+ }
+
+ public Query rewrite(IndexReader reader) throws IOException {
+ return this;
+ }
+
+ protected class FunctionWeight implements Weight {
+ Searcher searcher;
+ float queryNorm;
+ float queryWeight;
+
+ public FunctionWeight(Searcher searcher) {
+ this.searcher = searcher;
+ }
+
+ public Query getQuery() {
+ return FunctionQuery.this;
+ }
+
+ public float getValue() {
+ return queryWeight;
+ }
+
+ public float sumOfSquaredWeights() throws IOException {
+ queryWeight = getBoost();
+ return queryWeight * queryWeight;
+ }
+
+ public void normalize(float norm) {
+ this.queryNorm = norm;
+ queryWeight *= this.queryNorm;
+ }
+
+ public Scorer scorer(IndexReader reader) throws IOException {
+ return new AllScorer(getSimilarity(searcher), reader, this);
+ }
+
+ public Explanation explain(IndexReader reader, int doc) throws IOException {
+ return scorer(reader).explain(doc);
+ }
+ }
+
+ protected class AllScorer extends Scorer {
+ final IndexReader reader;
+ final FunctionWeight weight;
+ final int maxDoc;
+ final float qWeight;
+ int doc=-1;
+ final DocValues vals;
+
+ public AllScorer(Similarity similarity, IndexReader reader, FunctionWeight w) throws IOException {
+ super(similarity);
+ this.weight = w;
+ this.qWeight = w.getValue();
+ this.reader = reader;
+ this.maxDoc = reader.maxDoc();
+ vals = func.getValues(reader);
+ }
+
+ // instead of matching all docs, we could also embed a query.
+ // the score could either ignore the subscore, or boost it.
+ // Containment: floatline(foo:myTerm, "myFloatField", 1.0, 0.0f)
+ // Boost: foo:myTerm^floatline("myFloatField",1.0,0.0f)
+ public boolean next() throws IOException {
+ for(;;) {
+ ++doc;
+ if (doc>=maxDoc) {
+ return false;
+ }
+ if (reader.isDeleted(doc)) continue;
+ // todo: maybe allow score() to throw a specific exception
+ // and continue on to the next document if it is thrown...
+ // that may be useful, but exceptions aren't really good
+ // for flow control.
+ return true;
+ }
+ }
+
+ public int doc() {
+ return doc;
+ }
+
+ public float score() throws IOException {
+ return qWeight * vals.floatVal(doc);
+ }
+
+ public boolean skipTo(int target) throws IOException {
+ doc=target-1;
+ return next();
+ }
+
+ public Explanation explain(int doc) throws IOException {
+ float sc = qWeight * vals.floatVal(doc);
+
+ Explanation result = new Explanation();
+ result.setDescription("FunctionQuery(" + func
+ + "), product of:");
+ result.setValue(sc);
+ result.addDetail(vals.explain(doc));
+ result.addDetail(new Explanation(getBoost(), "boost"));
+ result.addDetail(new Explanation(weight.queryNorm,"queryNorm"));
+ return result;
+ }
+ }
+
+
+ protected Weight createWeight(Searcher searcher) {
+ return new FunctionQuery.FunctionWeight(searcher);
+ }
+
+
+ /** Prints a user-readable version of this query. */
+ public String toString(String field)
+ {
+ float boost = getBoost();
+ return (boost!=1.0?"(":"") + func.toString()
+ + (getBoost()==0 ? "" : ")^"+getBoost());
+ }
+
+
+ /** Returns true if <code>o</code> is equal to this. */
+ public boolean equals(Object o) {
+ if (FunctionQuery.class != o.getClass()) return false;
+ FunctionQuery other = (FunctionQuery)o;
+ return this.getBoost() == other.getBoost()
+ && this.func.equals(other.func);
+ }
+
+ /** Returns a hash code value for this object. */
+ public int hashCode() {
+ return func.hashCode() ^ Float.floatToIntBits(getBoost());
+ }
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/FunctionQuery.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/IntFieldSource.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/IntFieldSource.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/IntFieldSource.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/IntFieldSource.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.function.DocValues;
+import org.apache.lucene.search.function.ValueSource;
+import org.apache.lucene.search.FieldCache;
+
+import java.io.IOException;
+
+/**
+ * Obtains int field values from the {@link org.apache.lucene.search.FieldCache}
+ * using <code>getInts()</code>
+ * and makes those values available as other numeric types, casting as needed. *
+ * @author yonik
+ * @version $Id: IntFieldSource.java,v 1.2 2005/11/22 05:23:20 yonik Exp $
+ */
+
+public class IntFieldSource extends FieldCacheSource {
+ FieldCache.IntParser parser;
+
+ public IntFieldSource(String field) {
+ this(field, null);
+ }
+
+ public IntFieldSource(String field, FieldCache.IntParser parser) {
+ super(field);
+ this.parser = parser;
+ }
+
+ public String description() {
+ return "int(" + field + ')';
+ }
+
+ public DocValues getValues(IndexReader reader) throws IOException {
+ final int[] arr = (parser==null) ?
+ cache.getInts(reader, field) :
+ cache.getInts(reader, field, parser);
+ return new DocValues() {
+ public float floatVal(int doc) {
+ return (float)arr[doc];
+ }
+
+ public int intVal(int doc) {
+ return (int)arr[doc];
+ }
+
+ public long longVal(int doc) {
+ return (long)arr[doc];
+ }
+
+ public double doubleVal(int doc) {
+ return (double)arr[doc];
+ }
+
+ public String strVal(int doc) {
+ return Float.toString(arr[doc]);
+ }
+
+ public String toString(int doc) {
+ return description() + '=' + intVal(doc);
+ }
+
+ };
+ }
+
+ public boolean equals(Object o) {
+ if (o.getClass() != IntFieldSource.class) return false;
+ IntFieldSource other = (IntFieldSource)o;
+ return super.equals(other)
+ && this.parser==null ? other.parser==null :
+ this.parser.getClass() == other.parser.getClass();
+ }
+
+ public int hashCode() {
+ int h = parser==null ? Integer.class.hashCode() : parser.getClass().hashCode();
+ h += super.hashCode();
+ return h;
+ };
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/IntFieldSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/LinearFloatFunction.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/LinearFloatFunction.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/LinearFloatFunction.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/LinearFloatFunction.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.index.IndexReader;
+
+import java.io.IOException;
+
+/**
+ * <code>LinearFloatFunction</code> implements a linear function over
+ * another {@link ValueSource}.
+ * <br>
+ * Normally Used as an argument to a {@link FunctionQuery}
+ *
+ * @author yonik
+ * @version $Id: LinearFloatFunction.java,v 1.2 2005/11/22 05:23:21 yonik Exp $
+ */
+public class LinearFloatFunction extends ValueSource {
+ protected final ValueSource source;
+ protected final float slope;
+ protected final float intercept;
+
+ public LinearFloatFunction(ValueSource source, float slope, float intercept) {
+ this.source = source;
+ this.slope = slope;
+ this.intercept = intercept;
+ }
+
+ public String description() {
+ return slope + "*float(" + source.description() + ")+" + intercept;
+ }
+
+ public DocValues getValues(IndexReader reader) throws IOException {
+ final DocValues vals = source.getValues(reader);
+ return new DocValues() {
+ public float floatVal(int doc) {
+ return vals.floatVal(doc) * slope + intercept;
+ }
+ public int intVal(int doc) {
+ return (int)floatVal(doc);
+ }
+ public long longVal(int doc) {
+ return (long)floatVal(doc);
+ }
+ public double doubleVal(int doc) {
+ return (double)floatVal(doc);
+ }
+ public String strVal(int doc) {
+ return Float.toString(floatVal(doc));
+ }
+ public String toString(int doc) {
+ return slope + "*float(" + vals.toString(doc) + ")+" + intercept;
+ }
+ };
+ }
+
+ public int hashCode() {
+ int h = Float.floatToIntBits(slope);
+ h = (h >>> 2) | (h << 30);
+ h += Float.floatToIntBits(intercept);
+ h ^= (h << 14) | (h >>> 19);
+ return h + source.hashCode();
+ }
+
+ public boolean equals(Object o) {
+ if (LinearFloatFunction.class != o.getClass()) return false;
+ LinearFloatFunction other = (LinearFloatFunction)o;
+ return this.slope == other.slope
+ && this.intercept == other.intercept
+ && this.source.equals(other.source);
+ }
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/LinearFloatFunction.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/OrdFieldSource.java
URL: http://svn.apache.org/viewcvs/incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/OrdFieldSource.java?rev=372455&view=auto
==============================================================================
--- incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/OrdFieldSource.java (added)
+++ incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/OrdFieldSource.java Wed Jan 25 21:37:29 2006
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.function.DocValues;
+import org.apache.lucene.search.function.ValueSource;
+import org.apache.lucene.search.FieldCache;
+
+import java.io.IOException;
+
+/**
+ * Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getStringIndex().
+ * <br>
+ * The native lucene index order is used to assign an ordinal value for each field value.
+ * <br>Field values (terms) are lexicographically ordered by unicode value, and numbered starting at 1.
+ * <br>
+ * Example:<br>
+ * If there were only three field values: "apple","banana","pear"
+ * <br>then ord("apple")=1, ord("banana")=2, ord("pear")=3
+ * <p>
+ * WARNING: ord() depends on the position in an index and can thus change when other documents are inserted or deleted,
+ * or if a MultiSearcher is used.
+ * @author yonik
+ * @version $Id: OrdFieldSource.java,v 1.2 2005/11/22 05:23:21 yonik Exp $
+ */
+
+public class OrdFieldSource extends ValueSource {
+ protected String field;
+
+ public OrdFieldSource(String field) {
+ this.field = field;
+ }
+
+ public String description() {
+ return "ord(" + field + ')';
+ }
+
+ public DocValues getValues(IndexReader reader) throws IOException {
+ final int[] arr = FieldCache.DEFAULT.getStringIndex(reader, field).order;
+ return new DocValues() {
+ public float floatVal(int doc) {
+ return (float)arr[doc];
+ }
+
+ public int intVal(int doc) {
+ return (int)arr[doc];
+ }
+
+ public long longVal(int doc) {
+ return (long)arr[doc];
+ }
+
+ public double doubleVal(int doc) {
+ return (double)arr[doc];
+ }
+
+ public String strVal(int doc) {
+ // the string value of the ordinal, not the string itself
+ return Integer.toString(arr[doc]);
+ }
+
+ public String toString(int doc) {
+ return description() + '=' + intVal(doc);
+ }
+ };
+ }
+
+ public boolean equals(Object o) {
+ if (o.getClass() != OrdFieldSource.class) return false;
+ OrdFieldSource other = (OrdFieldSource)o;
+ return this.field.equals(field);
+ }
+
+ private static final int hcode = OrdFieldSource.class.hashCode();
+ public int hashCode() {
+ return hcode + field.hashCode();
+ };
+
+}
Propchange: incubator/solr/trunk/src/lucene_extras/org/apache/lucene/search/function/OrdFieldSource.java
------------------------------------------------------------------------------
svn:eol-style = native