You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by bu...@apache.org on 2010/07/27 22:44:39 UTC
svn commit: r979860 [5/5] - in /lucene/dev/branches/realtime_search: ./
lucene/ lucene/contrib/ lucene/contrib/highlighter/src/test/
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
lucene/contrib/memory/src/test/org/apache/lu...
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/_TestUtil.java Tue Jul 27 20:44:34 2010
@@ -23,6 +23,9 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.CheckIndex;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.Directory;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
@@ -38,20 +41,21 @@ public class _TestUtil {
public static void rmDir(File dir) throws IOException {
if (dir.exists()) {
- File[] files = dir.listFiles();
- for (int i = 0; i < files.length; i++) {
- if (!files[i].delete()) {
- throw new IOException("could not delete " + files[i]);
+ for (File f : dir.listFiles()) {
+ if (f.isDirectory()) {
+ rmDir(f);
+ } else {
+ if (!f.delete()) {
+ throw new IOException("could not delete " + f);
+ }
}
}
- dir.delete();
+ if (!dir.delete()) {
+ throw new IOException("could not delete " + dir);
+ }
}
}
- public static void rmDir(String dir) throws IOException {
- rmDir(new File(dir));
- }
-
public static void syncConcurrentMerges(IndexWriter writer) {
syncConcurrentMerges(writer.getConfig().getMergeScheduler());
}
@@ -78,39 +82,6 @@ public class _TestUtil {
return true;
}
- /** Use only for testing.
- * @deprecated -- in 3.0 we can use Arrays.toString
- * instead */
- @Deprecated
- public static String arrayToString(int[] array) {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for(int i=0;i<array.length;i++) {
- if (i > 0) {
- buf.append(" ");
- }
- buf.append(array[i]);
- }
- buf.append("]");
- return buf.toString();
- }
-
- /** Use only for testing.
- * @deprecated -- in 3.0 we can use Arrays.toString
- * instead */
- @Deprecated
- public static String arrayToString(Object[] array) {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for(int i=0;i<array.length;i++) {
- if (i > 0) {
- buf.append(" ");
- }
- buf.append(array[i]);
- }
- buf.append("]");
- return buf.toString();
- }
/** start and end are BOTH inclusive */
public static int nextInt(Random r, int start, int end) {
return start + r.nextInt(end-start+1);
@@ -130,7 +101,7 @@ public class _TestUtil {
final char[] buffer = new char[end];
for (int i = 0; i < end; i++) {
int t = r.nextInt(5);
- //buffer[i] = (char) (97 + r.nextInt(26));
+
if (0 == t && i < end - 1) {
// Make a surrogate pair
// High surrogate
@@ -162,7 +133,7 @@ public class _TestUtil {
0x3300, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA490, 0xA4D0, 0xA500, 0xA640,
0xA6A0, 0xA700, 0xA720, 0xA800, 0xA830, 0xA840, 0xA880, 0xA8E0, 0xA900,
0xA930, 0xA960, 0xA980, 0xAA00, 0xAA60, 0xAA80, 0xABC0, 0xAC00, 0xD7B0,
- 0xD800, 0xDB80, 0xDC00, 0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10,
+ 0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10,
0xFE20, 0xFE30, 0xFE50, 0xFE70, 0xFF00, 0xFFF0,
0x10000, 0x10080, 0x10100, 0x10140, 0x10190, 0x101D0, 0x10280, 0x102A0,
0x10300, 0x10330, 0x10380, 0x103A0, 0x10400, 0x10450, 0x10480, 0x10800,
@@ -188,7 +159,7 @@ public class _TestUtil {
0x33FF, 0x4DBF, 0x4DFF, 0x9FFF, 0xA48F, 0xA4CF, 0xA4FF, 0xA63F, 0xA69F,
0xA6FF, 0xA71F, 0xA7FF, 0xA82F, 0xA83F, 0xA87F, 0xA8DF, 0xA8FF, 0xA92F,
0xA95F, 0xA97F, 0xA9DF, 0xAA5F, 0xAA7F, 0xAADF, 0xABFF, 0xD7AF, 0xD7FF,
- 0xDB7F, 0xDBFF, 0xDFFF, 0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F,
+ 0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F,
0xFE2F, 0xFE4F, 0xFE6F, 0xFEFF, 0xFFEF, 0xFFFF,
0x1007F, 0x100FF, 0x1013F, 0x1018F, 0x101CF, 0x101FF, 0x1029F, 0x102DF,
0x1032F, 0x1034F, 0x1039F, 0x103DF, 0x1044F, 0x1047F, 0x104AF, 0x1083F,
@@ -213,9 +184,30 @@ public class _TestUtil {
return sb.toString();
}
- /** gets a random multiplier, which you should use when writing
- * random tests: multiply it by the number of iterations */
- public static int getRandomMultiplier() {
- return Integer.parseInt(System.getProperty("random.multiplier", "1"));
+ public static CodecProvider alwaysCodec(final Codec c) {
+ return new CodecProvider() {
+ @Override
+ public Codec getWriter(SegmentWriteState state) {
+ return c;
+ }
+
+ @Override
+ public Codec lookup(String name) {
+ // can't do this until we fix PreFlexRW to not
+ //impersonate PreFlex:
+ if (name.equals(c.name)) {
+ return c;
+ } else {
+ return CodecProvider.getDefault().lookup(name);
+ }
+ }
+ };
+ }
+
+ /** Return a CodecProvider that can read any of the
+ * default codecs, but always writes in the specified
+ * codec. */
+ public static CodecProvider alwaysCodec(final String codec) {
+ return alwaysCodec(CodecProvider.getDefault().lookup(codec));
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java Tue Jul 27 20:44:34 2010
@@ -40,7 +40,9 @@ public class AutomatonTestUtil {
if (!UnicodeUtil.validUTF16String(regexp))
continue;
try {
- return new RegExp(regexp, RegExp.NONE);
+ // NOTE: we parse-tostring-parse again, because we are
+ // really abusing RegExp.toString() here (its just for debugging)
+ return new RegExp(new RegExp(regexp, RegExp.NONE).toString(), RegExp.NONE);
} catch (Exception e) {}
}
}
@@ -75,12 +77,44 @@ public class AutomatonTestUtil {
return new String(buffer, 0, end);
}
- // picks a random int code point that this transition
- // accepts, avoiding the surrogates range since they are
- // "defined" in UTF32. Don't call this on a transition
- // that only accepts UTF16 surrogate values!!
+ // picks a random int code point, avoiding surrogates;
+ // throws IllegalArgumentException if this transition only
+ // accepts surrogates
private static int getRandomCodePoint(final Random r, final Transition t) {
- return t.min+r.nextInt(t.max-t.min+1);
+ final int code;
+ if (t.max < UnicodeUtil.UNI_SUR_HIGH_START ||
+ t.min > UnicodeUtil.UNI_SUR_HIGH_END) {
+ // easy: entire range is before or after surrogates
+ code = t.min+r.nextInt(t.max-t.min+1);
+ } else if (t.min >= UnicodeUtil.UNI_SUR_HIGH_START) {
+ if (t.max > UnicodeUtil.UNI_SUR_LOW_END) {
+ // after surrogates
+ code = 1+UnicodeUtil.UNI_SUR_LOW_END+r.nextInt(t.max-UnicodeUtil.UNI_SUR_LOW_END+1);
+ } else {
+ throw new IllegalArgumentException("transition accepts only surrogates: " + t);
+ }
+ } else if (t.max <= UnicodeUtil.UNI_SUR_LOW_END) {
+ if (t.min < UnicodeUtil.UNI_SUR_HIGH_START) {
+ // before surrogates
+ code = t.min + r.nextInt(UnicodeUtil.UNI_SUR_HIGH_START - t.min);
+ } else {
+ throw new IllegalArgumentException("transition accepts only surrogates: " + t);
+ }
+ } else {
+ // range includes all surrogates
+ int gap1 = UnicodeUtil.UNI_SUR_HIGH_START - t.min;
+ int gap2 = t.max - UnicodeUtil.UNI_SUR_LOW_END;
+ int c = r.nextInt(gap1+gap2);
+ if (c < gap1) {
+ code = t.min + c;
+ } else {
+ code = UnicodeUtil.UNI_SUR_LOW_END + c - gap1 + 1;
+ }
+ }
+
+ assert code >= t.min && code <= t.max && (code < UnicodeUtil.UNI_SUR_HIGH_START || code > UnicodeUtil.UNI_SUR_LOW_END):
+ "code=" + code + " min=" + t.min + " max=" + t.max;
+ return code;
}
public static class RandomAcceptedStrings {
@@ -204,7 +238,6 @@ public class AutomatonTestUtil {
} else {
t = s.transitionsArray[r.nextInt(s.numTransitions)];
}
-
soFar.add(getRandomCodePoint(r, t));
s = t.to;
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java Tue Jul 27 20:44:34 2010
@@ -19,7 +19,6 @@ package org.apache.lucene.util.automaton
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util._TestUtil;
import java.util.Random;
@@ -87,8 +86,8 @@ public class TestBasicOperations extends
public void testGetRandomAcceptedString() throws Throwable {
final Random r = newRandom();
- final int ITER1 = 100*_TestUtil.getRandomMultiplier();
- final int ITER2 = 100*_TestUtil.getRandomMultiplier();
+ final int ITER1 = 100 * RANDOM_MULTIPLIER;
+ final int ITER2 = 100 * RANDOM_MULTIPLIER;
for(int i=0;i<ITER1;i++) {
final RegExp re = AutomatonTestUtil.randomRegexp(r);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminism.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminism.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminism.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminism.java Tue Jul 27 20:44:34 2010
@@ -20,7 +20,6 @@ package org.apache.lucene.util.automaton
import java.util.Random;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
/**
* Not thorough, but tries to test determinism correctness
@@ -37,7 +36,8 @@ public class TestDeterminism extends Luc
/** test a bunch of random regular expressions */
public void testRegexps() throws Exception {
- for (int i = 0; i < 500*_TestUtil.getRandomMultiplier(); i++)
+ int num = 500 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++)
assertAutomaton(AutomatonTestUtil.randomRegexp(random).toAutomaton());
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java Tue Jul 27 20:44:34 2010
@@ -22,7 +22,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Random;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -37,7 +36,8 @@ public class TestDeterminizeLexicon exte
public void testLexicon() throws Exception {
random = newRandom();
- for (int i = 0; i < 3*_TestUtil.getRandomMultiplier(); i++) {
+ int num = 3 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++) {
automata.clear();
terms.clear();
for (int j = 0; j < 5000; j++) {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java Tue Jul 27 20:44:34 2010
@@ -46,15 +46,44 @@ public class TestUTF32ToUTF8 extends Luc
private void testOne(Random r, ByteRunAutomaton a, int startCode, int endCode, int iters) {
// Verify correct ints are accepted
+ final int nonSurrogateCount;
+ final boolean ovSurStart;
+ if (endCode < UnicodeUtil.UNI_SUR_HIGH_START ||
+ startCode > UnicodeUtil.UNI_SUR_LOW_END) {
+ // no overlap w/ surrogates
+ nonSurrogateCount = endCode - startCode + 1;
+ ovSurStart = false;
+ } else if (isSurrogate(startCode)) {
+ // start of range overlaps surrogates
+ nonSurrogateCount = endCode - startCode + 1 - (UnicodeUtil.UNI_SUR_LOW_END - startCode + 1);
+ ovSurStart = false;
+ } else if (isSurrogate(endCode)) {
+ // end of range overlaps surrogates
+ ovSurStart = true;
+ nonSurrogateCount = endCode - startCode + 1 - (endCode - UnicodeUtil.UNI_SUR_HIGH_START + 1);
+ } else {
+ // range completely subsumes surrogates
+ ovSurStart = true;
+ nonSurrogateCount = endCode - startCode + 1 - (UnicodeUtil.UNI_SUR_LOW_END - UnicodeUtil.UNI_SUR_HIGH_START + 1);
+ }
+
+ assert nonSurrogateCount > 0;
+
for(int iter=0;iter<iters;iter++) {
// pick random code point in-range
- final int code = _TestUtil.nextInt(r, startCode, endCode);
- if ((code >= UnicodeUtil.UNI_SUR_HIGH_START && code <= UnicodeUtil.UNI_SUR_HIGH_END) |
- (code >= UnicodeUtil.UNI_SUR_LOW_START && code <= UnicodeUtil.UNI_SUR_LOW_END)) {
- iter--;
- continue;
+ int code = startCode + r.nextInt(nonSurrogateCount);
+ if (isSurrogate(code)) {
+ if (ovSurStart) {
+ code = UnicodeUtil.UNI_SUR_LOW_END + 1 + (code - UnicodeUtil.UNI_SUR_HIGH_START);
+ } else {
+ code = UnicodeUtil.UNI_SUR_LOW_END + 1 + (code - startCode);
+ }
}
+
+ assert code >= startCode && code <= endCode: "code=" + code + " start=" + startCode + " end=" + endCode;
+ assert !isSurrogate(code);
+
assertTrue("DFA for range " + startCode + "-" + endCode + " failed to match code=" + code,
matches(a, code));
}
@@ -97,10 +126,14 @@ public class TestUTF32ToUTF8 extends Luc
}
}
+ private static boolean isSurrogate(int code) {
+ return code >= UnicodeUtil.UNI_SUR_HIGH_START && code <= UnicodeUtil.UNI_SUR_LOW_END;
+ }
+
public void testRandomRanges() throws Exception {
final Random r = random;
- int ITERS = 10*_TestUtil.getRandomMultiplier();
- int ITERS_PER_DFA = 100*_TestUtil.getRandomMultiplier();
+ int ITERS = 10 * RANDOM_MULTIPLIER;
+ int ITERS_PER_DFA = 100 * RANDOM_MULTIPLIER;
for(int iter=0;iter<ITERS;iter++) {
int x1 = getCodeStart(r);
int x2 = getCodeStart(r);
@@ -113,6 +146,11 @@ public class TestUTF32ToUTF8 extends Luc
startCode = x2;
endCode = x1;
}
+
+ if (isSurrogate(startCode) && isSurrogate(endCode)) {
+ iter--;
+ continue;
+ }
final Automaton a = new Automaton();
final State end = new State();
@@ -166,8 +204,10 @@ public class TestUTF32ToUTF8 extends Luc
}
public void testRandomRegexes() throws Exception {
- for (int i = 0; i < 250*_TestUtil.getRandomMultiplier(); i++)
+ int num = 250 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++) {
assertAutomaton(AutomatonTestUtil.randomRegexp(random).toAutomaton());
+ }
}
private void assertAutomaton(Automaton automaton) throws Exception {
@@ -175,7 +215,8 @@ public class TestUTF32ToUTF8 extends Luc
ByteRunAutomaton bra = new ByteRunAutomaton(automaton);
final AutomatonTestUtil.RandomAcceptedStrings ras = new AutomatonTestUtil.RandomAcceptedStrings(automaton);
- for (int i = 0; i < 1000*_TestUtil.getRandomMultiplier(); i++) {
+ int num = 1000 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++) {
final String string;
if (random.nextBoolean()) {
// likely not accepted
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java Tue Jul 27 20:44:34 2010
@@ -19,7 +19,6 @@ package org.apache.lucene.util.packed;
import org.apache.lucene.store.*;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import java.util.ArrayList;
import java.util.List;
@@ -52,7 +51,8 @@ public class TestPackedInts extends Luce
public void testPackedInts() throws IOException {
rnd = newRandom();
- for(int iter=0;iter<5*_TestUtil.getRandomMultiplier();iter++) {
+ int num = 5 * RANDOM_MULTIPLIER;
+ for (int iter = 0; iter < num; iter++) {
long ceil = 2;
for(int nbits=1;nbits<63;nbits++) {
final int valueCount = 100+rnd.nextInt(500);
Propchange: lucene/dev/branches/realtime_search/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,4 +1,5 @@
-/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java:953476-978809
+/lucene/dev/branches/preflexfixes/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java:967125-979432
+/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java:953476-979858
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:829134,829881,831036,896850,909334
Modified: lucene/dev/branches/realtime_search/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArrayMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArrayMap.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArrayMap.java (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArrayMap.java Tue Jul 27 20:44:34 2010
@@ -22,7 +22,6 @@ import java.util.*;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
public class TestCharArrayMap extends LuceneTestCase {
Random r = newRandom();
@@ -59,7 +58,8 @@ public class TestCharArrayMap extends Lu
}
public void testCharArrayMap() {
- for (int i=0; i<5*_TestUtil.getRandomMultiplier(); i++) { // pump this up for more random testing
+ int num = 5 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++) { // pump this up for more random testing
doRandom(1000,false);
doRandom(1000,true);
}
Propchange: lucene/dev/branches/realtime_search/solr/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,5 +1,6 @@
-/lucene/dev/branches/branch_3x/solr:949730,957490*,961612
-/lucene/dev/trunk/solr:953476-978809
+/lucene/dev/branches/branch_3x/solr:949730,957490*,961612,979161
+/lucene/dev/branches/preflexfixes/solr:967125-979432
+/lucene/dev/trunk/solr:953476-979858
/lucene/java/branches/lucene_2_4/solr:748824
/lucene/java/branches/lucene_2_9/solr:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr:818601-821336
Modified: lucene/dev/branches/realtime_search/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/CHANGES.txt?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/solr/CHANGES.txt Tue Jul 27 20:44:34 2010
@@ -203,6 +203,18 @@ New Features
http://wiki.apache.org/solr/SpatialSearch and the example. Refactored some items in Lucene spatial.
Removed SpatialTileField as the underlying CartesianTier is broken beyond repair and is going to be moved. (gsingers)
+* SOLR-2015: Add a boolean attribute autoGeneratePhraseQueries to TextField.
+ autoGeneratePhraseQueries="true" (the default) causes the query parser to
+ generate phrase queries if multiple tokens are generated from a single
+ non-quoted analysis string. For example WordDelimiterFilter splitting text:pdp-11
+ will cause the parser to generate text:"pdp 11" rather than (text:PDP OR text:11).
+ Note that autoGeneratePhraseQueries="true" tends to not work well for non whitespace
+ delimited languages. (yonik)
+
+* SOLR-1925: Add CSVResponseWriter (use wt=csv) that returns the list of documents
+ in CSV format. (Chris Mattmann, yonik)
+
+
Optimizations
----------------------
Propchange: lucene/dev/branches/realtime_search/solr/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/CHANGES.txt:949730,961612
+/lucene/dev/trunk/solr/CHANGES.txt:953476-979858
+/lucene/java/branches/lucene_2_4/solr/CHANGES.txt:748824
+/lucene/java/branches/lucene_2_9/solr/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/CHANGES.txt:818601-821336
+/lucene/java/branches/lucene_3_0/solr/CHANGES.txt:880793,896906
+/lucene/solr/trunk/CHANGES.txt:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/KEYS
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/KEYS:949730,961612
+/lucene/dev/trunk/solr/KEYS:953476-979858
+/lucene/java/branches/lucene_2_4/solr/KEYS:748824
+/lucene/java/branches/lucene_2_9/solr/KEYS:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/KEYS:818601-821336
+/lucene/java/branches/lucene_3_0/solr/KEYS:880793,896906
+/lucene/solr/trunk/KEYS:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/LICENSE.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/LICENSE.txt:949730,961612
+/lucene/dev/trunk/solr/LICENSE.txt:953476-979858
+/lucene/java/branches/lucene_2_4/solr/LICENSE.txt:748824
+/lucene/java/branches/lucene_2_9/solr/LICENSE.txt:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/LICENSE.txt:818601-821336
+/lucene/java/branches/lucene_3_0/solr/LICENSE.txt:880793,896906
+/lucene/solr/trunk/LICENSE.txt:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/NOTICE.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/NOTICE.txt:949730,961612
+/lucene/dev/trunk/solr/NOTICE.txt:953476-979858
+/lucene/java/branches/lucene_2_4/solr/NOTICE.txt:748824
+/lucene/java/branches/lucene_2_9/solr/NOTICE.txt:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/NOTICE.txt:818601-821336
+/lucene/java/branches/lucene_3_0/solr/NOTICE.txt:880793,896906
+/lucene/solr/trunk/NOTICE.txt:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/README.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/README.txt:949730,961612
+/lucene/dev/trunk/solr/README.txt:953476-979858
+/lucene/java/branches/lucene_2_4/solr/README.txt:748824
+/lucene/java/branches/lucene_2_9/solr/README.txt:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/README.txt:818601-821336
+/lucene/java/branches/lucene_3_0/solr/README.txt:880793,896906
+/lucene/solr/trunk/README.txt:922950-923910,923912-925091
Modified: lucene/dev/branches/realtime_search/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/build.xml?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/build.xml Tue Jul 27 20:44:34 2010
@@ -435,6 +435,7 @@
>
<sysproperty key="java.util.logging.config.file" value="${common-solr.dir}/testlogging.properties"/>
<sysproperty key="tests.luceneMatchVersion" value="${tests.luceneMatchVersion}"/>
+ <sysproperty key="tests.codec" value="${tests.codec}"/>
<jvmarg line="${dir.prop}"/>
<jvmarg line="${args}"/>
Propchange: lucene/dev/branches/realtime_search/solr/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/build.xml:949730,961612
+/lucene/dev/trunk/solr/build.xml:953476-979858
+/lucene/java/branches/lucene_2_4/solr/build.xml:748824
+/lucene/java/branches/lucene_2_9/solr/build.xml:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/build.xml:818601-821336
+/lucene/java/branches/lucene_3_0/solr/build.xml:880793,896906
+/lucene/solr/trunk/build.xml:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/client/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/client:949730,961612
+/lucene/dev/trunk/solr/client:953476-979858
+/lucene/java/branches/lucene_2_4/solr/client:748824
+/lucene/java/branches/lucene_2_9/solr/client:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/client:818601-821336
+/lucene/java/branches/lucene_3_0/solr/client:880793,896906
+/lucene/solr/trunk/client:922950-923910,923912-925091
Modified: lucene/dev/branches/realtime_search/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/common-build.xml?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/common-build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/common-build.xml Tue Jul 27 20:44:34 2010
@@ -44,6 +44,8 @@
<!-- TODO: measure toning this down by default to 1 -->
<property name="threadsPerProcessor" value="2"/>
+ <property name="tests.codec" value="random" />
+
<!-- Example directory -->
<property name="example" value="${common-solr.dir}/example" />
<!--
Propchange: lucene/dev/branches/realtime_search/solr/common-build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/common-build.xml:949730,961612
+/lucene/dev/trunk/solr/common-build.xml:953476-979858
+/lucene/java/branches/lucene_2_4/solr/common-build.xml:748824
+/lucene/java/branches/lucene_2_9/solr/common-build.xml:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/common-build.xml:818601-821336
+/lucene/java/branches/lucene_3_0/solr/common-build.xml:880793,896906
+/lucene/solr/trunk/common-build.xml:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/contrib:949730,961612
+/lucene/dev/trunk/solr/contrib:953476-979858
+/lucene/java/branches/lucene_2_4/solr/contrib:748824
+/lucene/java/branches/lucene_2_9/solr/contrib:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/contrib:818601-821336
+/lucene/java/branches/lucene_3_0/solr/contrib:880793,896906
+/lucene/solr/trunk/contrib:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/example/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/example:949730,961612
+/lucene/dev/trunk/solr/example:953476-979858
+/lucene/java/branches/lucene_2_4/solr/example:748824
+/lucene/java/branches/lucene_2_9/solr/example:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/example:818601-821336
+/lucene/java/branches/lucene_3_0/solr/example:880793,896906
+/lucene/solr/trunk/example:922950-923910,923912-925091
Modified: lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml (original)
+++ lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml Tue Jul 27 20:44:34 2010
@@ -213,8 +213,12 @@
words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
Synonyms and stopwords are customized by external files, and stemming is enabled.
+ The attribute autoGeneratePhraseQueries="true" (the default) causes words that get split to
+ form phrase queries. For example, WordDelimiterFilter splitting text:pdp-11 will cause the parser
+ to generate text:"pdp 11" rather than (text:PDP OR text:11).
+ NOTE: autoGeneratePhraseQueries="true" tends to not work well for non whitespace delimited languages.
-->
- <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<!-- in this example, we will only use synonyms at query time
Modified: lucene/dev/branches/realtime_search/solr/example/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/example/solr/conf/solrconfig.xml?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/example/solr/conf/solrconfig.xml (original)
+++ lucene/dev/branches/realtime_search/solr/example/solr/conf/solrconfig.xml Tue Jul 27 20:44:34 2010
@@ -1062,6 +1062,7 @@
<queryResponseWriter name="php" class="solr.PHPResponseWriter"/>
<queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/>
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter"/>
+ <queryResponseWriter name="csv" class="solr.CSVResponseWriter"/>
Custom response writers can be declared as needed...
Propchange: lucene/dev/branches/realtime_search/solr/lib/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/lib:949730,961612
+/lucene/dev/trunk/solr/lib:953476-979858
+/lucene/java/branches/lucene_2_4/solr/lib:748824
+/lucene/java/branches/lucene_2_9/solr/lib:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/lib:818601-821336
+/lucene/java/branches/lucene_3_0/solr/lib:880793,896906
+/lucene/solr/trunk/lib:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/lib/commons-httpclient-3.1.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar:949730,961612
-/lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar:953476-978809
+/lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/lib/commons-httpclient-3.1.jar:967125-979432
+/lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar:953476-979858
/lucene/solr/trunk/lib/commons-httpclient-3.1.jar:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/lib/jcl-over-slf4j-1.5.5.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar:949730,961612
-/lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar:953476-978809
+/lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/lib/jcl-over-slf4j-1.5.5.jar:967125-979432
+/lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar:953476-979858
/lucene/solr/trunk/lib/jcl-over-slf4j-1.5.5.jar:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/site/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/site:949730,961612
+/lucene/dev/trunk/solr/site:953476-979858
+/lucene/java/branches/lucene_2_4/solr/site:748824
+/lucene/java/branches/lucene_2_9/solr/site:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/site:818601-821336
+/lucene/java/branches/lucene_3_0/solr/site:880793,896906
+/lucene/solr/trunk/site:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/src/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/src:949730,961612
+/lucene/dev/trunk/solr/src:953476-979858
+/lucene/java/branches/lucene_2_4/solr/src:748824
+/lucene/java/branches/lucene_2_9/solr/src:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/src:818601-821336
+/lucene/java/branches/lucene_3_0/solr/src:880793,896906
+/lucene/solr/trunk/src:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common:949730,961612
-/lucene/dev/trunk/solr/src/common/org/apache/solr/common:953476-978809
+/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/src/common/org/apache/solr/common:967125-979432
+/lucene/dev/trunk/solr/src/common/org/apache/solr/common:953476-979858
/lucene/solr/trunk/src/common/org/apache/solr/common:922950-923910,923912-925091
Modified: lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/DateUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/DateUtil.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/DateUtil.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/DateUtil.java Tue Jul 27 20:44:34 2010
@@ -16,6 +16,8 @@ package org.apache.solr.common.util;
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Writer;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -196,5 +198,66 @@ public class DateUtil {
}
}
+ /** Formats the date and returns the calendar instance that was used (which may be reused) */
+ public static Calendar formatDate(Date date, Calendar cal, Appendable out) throws IOException {
+ // using a stringBuilder for numbers can be nice since
+ // a temporary string isn't used (it's added directly to the
+ // builder's buffer.
+
+ StringBuilder sb = out instanceof StringBuilder ? (StringBuilder)out : new StringBuilder();
+ if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);
+ cal.setTime(date);
+
+ int i = cal.get(Calendar.YEAR);
+ sb.append(i);
+ sb.append('-');
+ i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append('-');
+ i=cal.get(Calendar.DAY_OF_MONTH);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append('T');
+ i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append(':');
+ i=cal.get(Calendar.MINUTE);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append(':');
+ i=cal.get(Calendar.SECOND);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ i=cal.get(Calendar.MILLISECOND);
+ if (i != 0) {
+ sb.append('.');
+ if (i<100) sb.append('0');
+ if (i<10) sb.append('0');
+ sb.append(i);
+
+ // handle canonical format specifying fractional
+ // seconds shall not end in '0'. Given the slowness of
+ // integer div/mod, simply checking the last character
+ // is probably the fastest way to check.
+ int lastIdx = sb.length()-1;
+ if (sb.charAt(lastIdx)=='0') {
+ lastIdx--;
+ if (sb.charAt(lastIdx)=='0') {
+ lastIdx--;
+ }
+ sb.setLength(lastIdx+1);
+ }
+
+ }
+ sb.append('Z');
+
+ if (out != sb)
+ out.append(sb);
+
+ return cal;
+ }
+
}
\ No newline at end of file
Modified: lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/FastWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/FastWriter.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/FastWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/FastWriter.java Tue Jul 27 20:44:34 2010
@@ -27,9 +27,9 @@ public class FastWriter extends Writer {
// use default BUFSIZE of BufferedWriter so if we wrap that
// it won't cause double buffering.
private static final int BUFSIZE = 8192;
- private final Writer sink;
- private final char[] buf;
- private int pos;
+ protected final Writer sink;
+ protected final char[] buf;
+ protected int pos;
public FastWriter(Writer w) {
this(w, new char[BUFSIZE], 0);
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java Tue Jul 27 20:44:34 2010
@@ -24,7 +24,6 @@ import java.io.Reader;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.compound.CompoundWordTokenFilterBase;
-import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -36,7 +35,7 @@ import org.apache.solr.util.plugin.Resou
import java.util.Map;
/**
- * Factory for {@link DictionaryCompoundWordTokenFilter}
+ * Factory for {@link HyphenationCompoundWordTokenFilter}
* <p>
* This factory accepts the following parameters:
* <ul>
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/SolrCore.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/SolrCore.java Tue Jul 27 20:44:34 2010
@@ -34,6 +34,7 @@ import org.apache.solr.handler.component
import org.apache.solr.highlight.DefaultSolrHighlighter;
import org.apache.solr.highlight.SolrHighlighter;
import org.apache.solr.request.*;
+import org.apache.solr.response.*;
import org.apache.solr.response.BinaryResponseWriter;
import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.PHPResponseWriter;
@@ -43,7 +44,6 @@ import org.apache.solr.response.QueryRes
import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.RubyResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.response.VelocityResponseWriter;
import org.apache.solr.response.XMLResponseWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.QParserPlugin;
@@ -1408,6 +1408,7 @@ public final class SolrCore implements S
m.put("raw", new RawResponseWriter());
m.put("javabin", new BinaryResponseWriter());
m.put("velocity", new VelocityResponseWriter());
+ m.put("csv", new CSVResponseWriter());
DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue Jul 27 20:44:34 2010
@@ -70,18 +70,18 @@ public class CSVRequestHandler extends C
abstract class CSVLoader extends ContentStreamLoader {
- static String SEPARATOR="separator";
- static String FIELDNAMES="fieldnames";
- static String HEADER="header";
- static String SKIP="skip";
- static String SKIPLINES="skipLines";
- static String MAP="map";
- static String TRIM="trim";
- static String EMPTY="keepEmpty";
- static String SPLIT="split";
- static String ENCAPSULATOR="encapsulator";
- static String ESCAPE="escape";
- static String OVERWRITE="overwrite";
+ public static final String SEPARATOR="separator";
+ public static final String FIELDNAMES="fieldnames";
+ public static final String HEADER="header";
+ public static final String SKIP="skip";
+ public static final String SKIPLINES="skipLines";
+ public static final String MAP="map";
+ public static final String TRIM="trim";
+ public static final String EMPTY="keepEmpty";
+ public static final String SPLIT="split";
+ public static final String ENCAPSULATOR="encapsulator";
+ public static final String ESCAPE="escape";
+ public static final String OVERWRITE="overwrite";
private static Pattern colonSplit = Pattern.compile(":");
private static Pattern commaSplit = Pattern.compile(",");
@@ -219,7 +219,7 @@ abstract class CSVLoader extends Content
// if only encapsulator or escape is set, disable the other escaping mechanism
if (encapsulator == null && escape != null) {
- strategy.setEncapsulator((char)-2); // TODO: add CSVStrategy.ENCAPSULATOR_DISABLED
+ strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
strategy.setEscape(escape.charAt(0));
} else {
if (encapsulator != null) {
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java Tue Jul 27 20:44:34 2010
@@ -1126,10 +1126,6 @@ class TermIndex {
return br;
}
- public BytesRef skipTo(Term target) throws IOException {
- throw new UnsupportedOperationException();
- }
-
public BytesRef skipTo(int termNumber) throws IOException {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java Tue Jul 27 20:44:34 2010
@@ -46,13 +46,21 @@ import java.io.StringReader;
* @version $Id$
*/
public class TextField extends FieldType {
+ protected boolean autoGeneratePhraseQueries = true;
+
protected void init(IndexSchema schema, Map<String,String> args) {
properties |= TOKENIZED;
if (schema.getVersion()> 1.1f) properties &= ~OMIT_TF_POSITIONS;
-
+ String autoGeneratePhraseQueriesStr = args.remove("autoGeneratePhraseQueries");
+ if (autoGeneratePhraseQueriesStr != null)
+ autoGeneratePhraseQueries = Boolean.parseBoolean(autoGeneratePhraseQueriesStr);
super.init(schema, args);
}
+ public boolean getAutoGeneratePhraseQueries() {
+ return autoGeneratePhraseQueries;
+ }
+
public SortField getSortField(SchemaField field, boolean reverse) {
return getStringSort(field, reverse);
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/SolrQueryParser.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/SolrQueryParser.java Tue Jul 27 20:44:34 2010
@@ -142,11 +142,15 @@ public class SolrQueryParser extends Que
return parser.subQuery(queryText, null).getQuery();
}
}
- //Intercept poly fields, as they get expanded by default to an OR clause of
- SchemaField sf = schema.getField(field);
- //TODO: is there anyway to avoid this instance of check?
- if (sf != null&& !(sf.getType() instanceof TextField)){//we have a poly field, deal with it specially by delegating to the FieldType
- return sf.getType().getFieldQuery(parser, sf, queryText);
+ SchemaField sf = schema.getFieldOrNull(field);
+ if (sf != null) {
+ FieldType ft = sf.getType();
+ // delegate to type for everything except TextField
+ if (ft instanceof TextField) {
+ return super.getFieldQuery(field, queryText, quoted || ((TextField)ft).getAutoGeneratePhraseQueries());
+ } else {
+ return sf.getType().getFieldQuery(parser, sf, queryText);
+ }
}
// default to a normal field query
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/util/SolrPluginUtils.java Tue Jul 27 20:44:34 2010
@@ -205,7 +205,7 @@ public class SolrPluginUtils {
// TODO - should field order be maintained?
String[] flst = split(fl);
if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) {
- Set<String> set = new HashSet<String>();
+ Set<String> set = new LinkedHashSet<String>();
for (String fname : flst) {
if("score".equalsIgnoreCase(fname))
flags |= SolrIndexSearcher.GET_SCORES;
Propchange: lucene/dev/branches/realtime_search/solr/src/maven/solr-core-pom.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template:949730,961612
-/lucene/dev/trunk/solr/src/maven/solr-core-pom.xml.template:953476-978809
+/lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/src/maven/solr-core-pom.xml.template:967125-979432
+/lucene/dev/trunk/solr/src/maven/solr-core-pom.xml.template:953476-979858
/lucene/solr/trunk/src/maven/solr-core-pom.xml.template:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/src/maven/solr-solrj-pom.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template:949730,961612
-/lucene/dev/trunk/solr/src/maven/solr-solrj-pom.xml.template:953476-978809
+/lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/src/maven/solr-solrj-pom.xml.template:967125-979432
+/lucene/dev/trunk/solr/src/maven/solr-solrj-pom.xml.template:953476-979858
/lucene/solr/trunk/src/maven/solr-solrj-pom.xml.template:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/src/solrj/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/src/solrj/org:949730,961612
-/lucene/dev/trunk/solr/src/solrj/org:953476-978809
+/lucene/dev/branches/branch_3x/solr/src/solrj/org:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/src/solrj/org:967125-979432
+/lucene/dev/trunk/solr/src/solrj/org:953476-979858
/lucene/solr/trunk/src/solrj/org:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/src/test/org/apache/solr/client/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/client:949730,961612
-/lucene/dev/trunk/solr/src/test/org/apache/solr/client:953476-978809
+/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/client:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/src/test/org/apache/solr/client:967125-979432
+/lucene/dev/trunk/solr/src/test/org/apache/solr/client:953476-979858
/lucene/solr/trunk/src/test/org/apache/solr/client:922950-923910,923912-925091
Modified: lucene/dev/branches/realtime_search/solr/src/test/test-files/solr/conf/schema12.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/test/test-files/solr/conf/schema12.xml?rev=979860&r1=979859&r2=979860&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/test/test-files/solr/conf/schema12.xml (original)
+++ lucene/dev/branches/realtime_search/solr/src/test/test-files/solr/conf/schema12.xml Tue Jul 27 20:44:34 2010
@@ -145,6 +145,35 @@
</fieldType>
+ <!-- field type that doesn't generate phrases from unquoted multiple tokens per analysis unit -->
+ <fieldType name="text_np" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false" >
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
<fieldtype name="nametext" class="solr.TextField">
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldtype>
@@ -403,6 +432,8 @@
<field name="weight" type="float" indexed="true" stored="true"/>
<field name="bday" type="date" indexed="true" stored="true"/>
+ <field name="text_np" type="text_np" indexed="true" stored="false"/>
+
<field name="title_stemmed" type="text" indexed="true" stored="false"/>
<field name="title_lettertok" type="lettertok" indexed="true" stored="false"/>
Propchange: lucene/dev/branches/realtime_search/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -1,3 +1,4 @@
-/lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:949730,961612
-/lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:953476-978809
+/lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:949730,961612,979161
+/lucene/dev/branches/preflexfixes/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:967125-979432
+/lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:953476-979858
/lucene/solr/trunk/src/webapp/src/org/apache/solr/client/solrj/embedded:922950-923910,923912-925091
Propchange: lucene/dev/branches/realtime_search/solr/testlogging.properties
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Jul 27 20:44:34 2010
@@ -0,0 +1,7 @@
+/lucene/dev/branches/branch_3x/solr/testlogging.properties:949730,961612
+/lucene/dev/trunk/solr/testlogging.properties:953476-979858
+/lucene/java/branches/lucene_2_4/solr/testlogging.properties:748824
+/lucene/java/branches/lucene_2_9/solr/testlogging.properties:817269-818600,825998,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9_back_compat_tests/solr/testlogging.properties:818601-821336
+/lucene/java/branches/lucene_3_0/solr/testlogging.properties:880793,896906
+/lucene/solr/trunk/testlogging.properties:922950-923910,923912-925091