You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2012/03/06 19:13:50 UTC
svn commit: r1297628 [10/13] - in /lucene/dev/branches/solr_3159_jetty8: ./
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/maven/lucene/contrib/demo/
dev-tools/maven/lucene/contrib/highlighter/
dev-tools/maven/lucene/contrib/memory/ dev-tools/maven...
Modified: lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java Tue Mar 6 18:13:38 2012
@@ -28,7 +28,7 @@ import org.apache.lucene.util.*;
* Unit tests for {@link FSTCompletion}.
*/
public class FSTCompletionTest extends LuceneTestCase {
- public static TermFreq tf(String t, float v) {
+ public static TermFreq tf(String t, int v) {
return new TermFreq(t, v);
}
@@ -62,28 +62,28 @@ public class FSTCompletionTest extends L
tf("foundation", 1),
tf("fourblah", 1),
tf("fourteen", 1),
- tf("four", 0f),
- tf("fourier", 0f),
- tf("fourty", 0f),
+ tf("four", 0),
+ tf("fourier", 0),
+ tf("fourty", 0),
tf("xo", 1),
};
return keys;
}
public void testExactMatchHighPriority() throws Exception {
- assertMatchEquals(completion.lookup("two", 1),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("two", random), 1),
"two/1.0");
}
public void testExactMatchLowPriority() throws Exception {
- assertMatchEquals(completion.lookup("one", 2),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("one", random), 2),
"one/0.0",
"oneness/1.0");
}
public void testExactMatchReordering() throws Exception {
// Check reordering of exact matches.
- assertMatchEquals(completion.lookup("four", 4),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("four", random), 4),
"four/0.0",
"fourblah/1.0",
"fourteen/1.0",
@@ -92,49 +92,49 @@ public class FSTCompletionTest extends L
public void testRequestedCount() throws Exception {
// 'one' is promoted after collecting two higher ranking results.
- assertMatchEquals(completion.lookup("one", 2),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("one", random), 2),
"one/0.0",
"oneness/1.0");
// 'four' is collected in a bucket and then again as an exact match.
- assertMatchEquals(completion.lookup("four", 2),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("four", random), 2),
"four/0.0",
"fourblah/1.0");
// Check reordering of exact matches.
- assertMatchEquals(completion.lookup("four", 4),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("four", random), 4),
"four/0.0",
"fourblah/1.0",
"fourteen/1.0",
"fourier/0.0");
// 'one' is at the top after collecting all alphabetical results.
- assertMatchEquals(completionAlphabetical.lookup("one", 2),
+ assertMatchEquals(completionAlphabetical.lookup(_TestUtil.stringToCharSequence("one", random), 2),
"one/0.0",
"oneness/1.0");
// 'one' is not promoted after collecting two higher ranking results.
FSTCompletion noPromotion = new FSTCompletion(completion.getFST(), true, false);
- assertMatchEquals(noPromotion.lookup("one", 2),
+ assertMatchEquals(noPromotion.lookup(_TestUtil.stringToCharSequence("one", random), 2),
"oneness/1.0",
"onerous/1.0");
// 'one' is at the top after collecting all alphabetical results.
- assertMatchEquals(completionAlphabetical.lookup("one", 2),
+ assertMatchEquals(completionAlphabetical.lookup(_TestUtil.stringToCharSequence("one", random), 2),
"one/0.0",
"oneness/1.0");
}
public void testMiss() throws Exception {
- assertMatchEquals(completion.lookup("xyz", 1));
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("xyz", random), 1));
}
public void testAlphabeticWithWeights() throws Exception {
- assertEquals(0, completionAlphabetical.lookup("xyz", 1).size());
+ assertEquals(0, completionAlphabetical.lookup(_TestUtil.stringToCharSequence("xyz", random), 1).size());
}
public void testFullMatchList() throws Exception {
- assertMatchEquals(completion.lookup("one", Integer.MAX_VALUE),
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("one", random), Integer.MAX_VALUE),
"oneness/1.0",
"onerous/1.0",
"onesimus/1.0",
@@ -148,7 +148,7 @@ public class FSTCompletionTest extends L
builder.add(new BytesRef(key), 0);
FSTCompletion lookup = builder.build();
- List<Completion> result = lookup.lookup(key, 1);
+ List<Completion> result = lookup.lookup(_TestUtil.stringToCharSequence(key, random), 1);
assertEquals(1, result.size());
}
@@ -158,16 +158,16 @@ public class FSTCompletionTest extends L
Random r = random;
List<TermFreq> keys = new ArrayList<TermFreq>();
for (int i = 0; i < 5000; i++) {
- keys.add(new TermFreq(_TestUtil.randomSimpleString(r), -1.0f));
+ keys.add(new TermFreq(_TestUtil.randomSimpleString(r), -1));
}
lookup.build(new TermFreqArrayIterator(keys));
// All the weights were constant, so all returned buckets must be constant, whatever they
// are.
- Float previous = null;
+ Long previous = null;
for (TermFreq tf : keys) {
- Float current = lookup.get(tf.term.utf8ToString());
+ Long current = ((Number)lookup.get(_TestUtil.bytesToCharSequence(tf.term, random))).longValue();
if (previous != null) {
assertEquals(previous, current);
}
@@ -180,28 +180,27 @@ public class FSTCompletionTest extends L
FSTCompletionLookup lookup = new FSTCompletionLookup();
lookup.build(new TermFreqArrayIterator(input));
-
for (TermFreq tf : input) {
- assertTrue("Not found: " + tf.term, lookup.get(tf.term.utf8ToString()) != null);
- assertEquals(tf.term.utf8ToString(), lookup.lookup(tf.term.utf8ToString(), true, 1).get(0).key);
+ assertNotNull("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random)));
+ assertEquals(tf.term.utf8ToString(), lookup.lookup(_TestUtil.bytesToCharSequence(tf.term, random), true, 1).get(0).key.toString());
}
- List<LookupResult> result = lookup.lookup("wit", true, 5);
+ List<LookupResult> result = lookup.lookup(_TestUtil.stringToCharSequence("wit", random), true, 5);
assertEquals(5, result.size());
- assertTrue(result.get(0).key.equals("wit")); // exact match.
- assertTrue(result.get(1).key.equals("with")); // highest count.
+ assertTrue(result.get(0).key.toString().equals("wit")); // exact match.
+ assertTrue(result.get(1).key.toString().equals("with")); // highest count.
}
public void testEmptyInput() throws Exception {
completion = new FSTCompletionBuilder().build();
- assertMatchEquals(completion.lookup("", 10));
+ assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("", random), 10));
}
public void testRandom() throws Exception {
List<TermFreq> freqs = new ArrayList<TermFreq>();
Random rnd = random;
for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
- float weight = rnd.nextFloat() * 100;
+ int weight = random.nextInt(100);
freqs.add(new TermFreq("" + rnd.nextLong(), weight));
}
@@ -212,8 +211,8 @@ public class FSTCompletionTest extends L
final String term = tf.term.utf8ToString();
for (int i = 1; i < term.length(); i++) {
String prefix = term.substring(0, i);
- for (LookupResult lr : lookup.lookup(prefix, true, 10)) {
- assertTrue(lr.key.startsWith(prefix));
+ for (LookupResult lr : lookup.lookup(_TestUtil.stringToCharSequence(prefix, random), true, 10)) {
+ assertTrue(lr.key.toString().startsWith(prefix));
}
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java Tue Mar 6 18:13:38 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.search.suggest
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import org.apache.lucene.search.suggest.fst.Sort.BufferSize;
import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter;
@@ -61,7 +62,7 @@ public class TestSort extends LuceneTest
@Test
public void testIntermediateMerges() throws Exception {
// Sort 20 mb worth of data with 1mb buffer, binary merging.
- SortInfo info = checkSort(new Sort(BufferSize.megabytes(1), Sort.defaultTempDir(), 2),
+ SortInfo info = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), 2),
generateRandom(Sort.MB * 20));
assertTrue(info.mergeRounds > 10);
}
@@ -69,7 +70,7 @@ public class TestSort extends LuceneTest
@Test
public void testSmallRandom() throws Exception {
// Sort 20 mb worth of data with 1mb buffer.
- SortInfo sortInfo = checkSort(new Sort(BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
+ SortInfo sortInfo = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
generateRandom(Sort.MB * 20));
assertEquals(1, sortInfo.mergeRounds);
}
@@ -77,7 +78,7 @@ public class TestSort extends LuceneTest
@Test @Nightly
public void testLargerRandom() throws Exception {
// Sort 100MB worth of data with 15mb buffer.
- checkSort(new Sort(BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
+ checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
generateRandom(Sort.MB * 100));
}
@@ -92,14 +93,25 @@ public class TestSort extends LuceneTest
byte [][] bytes = data.toArray(new byte[data.size()][]);
return bytes;
}
-
+
+ static final Comparator<byte[]> unsignedByteOrderComparator = new Comparator<byte[]>() {
+ public int compare(byte[] left, byte[] right) {
+ final int max = Math.min(left.length, right.length);
+ for (int i = 0, j = 0; i < max; i++, j++) {
+ int diff = (left[i] & 0xff) - (right[j] & 0xff);
+ if (diff != 0)
+ return diff;
+ }
+ return left.length - right.length;
+ }
+ };
/**
* Check sorting data on an instance of {@link Sort}.
*/
private SortInfo checkSort(Sort sort, byte[][] data) throws IOException {
File unsorted = writeAll("unsorted", data);
- Arrays.sort(data, Sort.unsignedByteOrderComparator);
+ Arrays.sort(data, unsignedByteOrderComparator);
File golden = writeAll("golden", data);
File sorted = new File(tempDir, "sorted");
Modified: lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Tue Mar 6 18:13:38 2012
@@ -45,33 +45,33 @@ public class WFSTCompletionTest extends
suggester.build(new TermFreqArrayIterator(keys));
// top N of 2, but only foo is available
- List<LookupResult> results = suggester.lookup("f", false, 2);
+ List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random), false, 2);
assertEquals(1, results.size());
- assertEquals("foo", results.get(0).key);
+ assertEquals("foo", results.get(0).key.toString());
assertEquals(50, results.get(0).value, 0.01F);
// top N of 1 for 'bar': we return this even though barbar is higher
- results = suggester.lookup("bar", false, 1);
+ results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random), false, 1);
assertEquals(1, results.size());
- assertEquals("bar", results.get(0).key);
+ assertEquals("bar", results.get(0).key.toString());
assertEquals(10, results.get(0).value, 0.01F);
// top N Of 2 for 'b'
- results = suggester.lookup("b", false, 2);
+ results = suggester.lookup(_TestUtil.stringToCharSequence("b", random), false, 2);
assertEquals(2, results.size());
- assertEquals("barbar", results.get(0).key);
+ assertEquals("barbar", results.get(0).key.toString());
assertEquals(12, results.get(0).value, 0.01F);
- assertEquals("bar", results.get(1).key);
+ assertEquals("bar", results.get(1).key.toString());
assertEquals(10, results.get(1).value, 0.01F);
// top N of 3 for 'ba'
- results = suggester.lookup("ba", false, 3);
+ results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random), false, 3);
assertEquals(3, results.size());
- assertEquals("barbar", results.get(0).key);
+ assertEquals("barbar", results.get(0).key.toString());
assertEquals(12, results.get(0).value, 0.01F);
- assertEquals("bar", results.get(1).key);
+ assertEquals("bar", results.get(1).key.toString());
assertEquals(10, results.get(1).value, 0.01F);
- assertEquals("barbara", results.get(2).key);
+ assertEquals("barbara", results.get(2).key.toString());
assertEquals(6, results.get(2).value, 0.01F);
}
@@ -100,7 +100,7 @@ public class WFSTCompletionTest extends
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random.nextInt(1<<24);
slowCompletor.put(s, (long)weight);
- keys[i] = new TermFreq(s, (float) weight);
+ keys[i] = new TermFreq(s, weight);
}
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
@@ -109,7 +109,7 @@ public class WFSTCompletionTest extends
for (String prefix : allPrefixes) {
final int topN = _TestUtil.nextInt(random, 1, 10);
- List<LookupResult> r = suggester.lookup(prefix, false, topN);
+ List<LookupResult> r = suggester.lookup(_TestUtil.stringToCharSequence(prefix, random), false, topN);
// 2. go thru whole treemap (slowCompletor) and check its actually the best suggestion
final List<LookupResult> matches = new ArrayList<LookupResult>();
@@ -117,7 +117,7 @@ public class WFSTCompletionTest extends
// TODO: could be faster... but its slowCompletor for a reason
for (Map.Entry<String,Long> e : slowCompletor.entrySet()) {
if (e.getKey().startsWith(prefix)) {
- matches.add(new LookupResult(e.getKey(), (float)e.getValue().longValue()));
+ matches.add(new LookupResult(e.getKey(), e.getValue().longValue()));
}
}
@@ -126,7 +126,7 @@ public class WFSTCompletionTest extends
public int compare(LookupResult left, LookupResult right) {
int cmp = Float.compare(right.value, left.value);
if (cmp == 0) {
- return left.key.compareTo(right.key);
+ return left.compareTo(right);
} else {
return cmp;
}
@@ -140,7 +140,7 @@ public class WFSTCompletionTest extends
for(int hit=0;hit<r.size();hit++) {
//System.out.println(" check hit " + hit);
- assertEquals(matches.get(hit).key, r.get(hit).key);
+ assertEquals(matches.get(hit).key.toString(), r.get(hit).key.toString());
assertEquals(matches.get(hit).value, r.get(hit).value, 0f);
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/CHANGES.txt?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/CHANGES.txt (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/CHANGES.txt Tue Mar 6 18:13:38 2012
@@ -104,6 +104,8 @@ New Features
(Stefan Matheis, Mark Miller)
SOLR-3108: Error in SolrCloud's replica lookup code when replica's are hosted in same Solr instance.
(Bruno Dumon, Sami Siren, Mark Miller)
+ SOLR-3080: Remove shard info from zookeeper when SolrCore is explicitlyunloaded.
+ (yonik, Mark Miller, siren)
* SOLR-1566: Transforming documents in the ResponseWriters. This will allow
for more complex results in responses and open the door for function queries
@@ -227,7 +229,8 @@ New Features
* SOLR-2459: Expose LogLevel selection with a RequestHandler rather then servlet
(Stefan Matheis, Upayavira, ryan)
-* SOLR-3134: Include shard info in distributed response when shards.info=true (ryan)
+* SOLR-3134: Include shard info in distributed response when shards.info=true
+ (Russell Black, ryan)
Optimizations
@@ -261,6 +264,9 @@ Optimizations
Bug Fixes
----------------------
+* SOLR-3165: Cannot use DIH in Solrcloud + Zookeeper (Alexey Serba,
+ Mark Miller, siren)
+
* SOLR-3068: Occasional NPE in ThreadDumpHandler (siren)
* SOLR-2762: FSTLookup could return duplicate results or one results less
@@ -432,6 +438,9 @@ Other Changes
* SOLR-2607: Removed obsolete client/ folder (ehatcher, Eric Pugh, janhoy)
+* SOLR-3202: Dropping Support for JSP. New Admin UI is all client side (ryan)
+
+
Documentation
----------------------
@@ -453,6 +462,9 @@ Upgrading from Solr 3.5
HTMLStripCharFilter in Solr version 3.5 and earlier: the old implementation
(bugs and all) is preserved as LegacyHTMLStripCharFilter.
+* SOLR-3040: The DIH's admin UI (dataimport.jsp) now requires DIH request handlers to start with
+ a '/'. (dsmiley)
+
New Features
----------------------
* SOLR-2904: BinaryUpdateRequestHandler should be able to accept multiple update requests from
@@ -515,6 +527,9 @@ New Features
* SOLR-3143: Add SuggestQueryConverter, a QueryConverter intended for
auto-suggesters. (Robert Muir)
+* SOLR-3033: ReplicationHandler's backup command now supports a 'maxNumberOfBackups'
+ init param that can be used to delete all but the most recent N backups. (Torsten Krah, James Dyer)
+
Optimizations
----------------------
* SOLR-1931: Speedup for LukeRequestHandler and admin/schema browser. New parameter
@@ -531,6 +546,8 @@ Optimizations
Bug Fixes
----------------------
+* SOLR-3187 SystemInfoHandler leaks filehandles (siren)
+
* SOLR-2912: Fixed File descriptor leak in ShowFileRequestHandler (Michael Ryan, shalin)
* SOLR-2819: Improved speed of parsing hex entities in HTMLStripCharFilter
@@ -616,6 +633,12 @@ Bug Fixes
* SOLR-2909: Add support for ResourceLoaderAware tokenizerFactories in synonym
filter factories. (Tom Klonikowski, Jun Ohtani via Koji Sekiguchi)
+
+* SOLR-3168: ReplicationHandler "numberToKeep" & "maxNumberOfBackups" parameters
+ would keep only 1 backup, even if more than 1 was specified (Neil Hooey, James Dyer)
+
+* SOLR-3195: timeAllowed is ignored for grouping queries
+ (Russell Black via Martijn van Groningen)
Other Changes
----------------------
@@ -676,7 +699,7 @@ New Features
(Chris Male, Mark Holland, Gunnlaugur Thor Briem, Ryan McKinley)
* SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep'
- param that can be used to delete all but the most recent N backups.
+ request param that can be used to delete all but the most recent N backups.
(James Dyer via hossman)
Optimizations
Modified: lucene/dev/branches/solr_3159_jetty8/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/build.xml?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/build.xml (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/build.xml Tue Mar 6 18:13:38 2012
@@ -130,7 +130,7 @@
<target name="compile" description="Compile the source code."
depends="compile-core, compile-contrib"/>
<target name="test" description="Validate, then run core, solrj, and contrib unit tests."
- depends="validate, test-jsp, test-core, test-contrib"/>
+ depends="validate, test-core, test-contrib"/>
<target name="test-core" description="Runs the core and solrj unit tests."
depends="test-solr-core, test-solrj"/>
<target name="compile-test" description="Compile unit tests."
@@ -169,14 +169,7 @@
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
</target>
-
- <!-- Webapp targets -->
- <target name="test-jsp">
- <ant dir="webapp" target="test" inheritall="false">
- <propertyset refid="uptodate.and.compiled.properties"/>
- </ant>
- </target>
-
+
<!-- Validation (license/ notice checks). -->
<target name="validate" depends="compile-tools" description="Validate legal stuff.">
<license-check-macro dir="${basedir}">
Modified: lucene/dev/branches/solr_3159_jetty8/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/common-build.xml?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/common-build.xml (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/common-build.xml Tue Mar 6 18:13:38 2012
@@ -49,8 +49,8 @@
<property name="tests.loggingfile" value="${common-solr.dir}/testlogging.properties"/>
<property name="tests.cleanthreads.sysprop" value="perClass"/>
- <property name="clover.db.dir" location="${dest}/tests/clover/db"/>
- <property name="clover.report.dir" location="${dest}/tests/clover/reports"/>
+ <property name="clover.db.dir" location="${dest}/test/clover/db"/>
+ <property name="clover.report.dir" location="${dest}/test/clover/reports"/>
<available property="clover.present" classname="com.cenqua.clover.tasks.CloverReportTask"/>
<condition property="clover.enabled">
<and>
@@ -83,6 +83,7 @@
<pathelement path="${analyzers-common.jar}"/>
<pathelement path="${analyzers-kuromoji.jar}"/>
<pathelement path="${analyzers-phonetic.jar}"/>
+ <pathelement path="${analyzers-uima.jar}"/>
<pathelement path="${highlighter.jar}"/>
<pathelement path="${memory.jar}"/>
<pathelement path="${misc.jar}"/>
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java Tue Mar 6 18:13:38 2012
@@ -19,7 +19,9 @@ package org.apache.solr.handler.dataimpo
import com.sun.mail.imap.IMAPMessage;
import org.apache.tika.Tika;
+import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaMetadataKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,8 +171,8 @@ public class MailEntityProcessor extends
InputStream is = part.getInputStream();
String fileName = part.getFileName();
Metadata md = new Metadata();
- md.set(Metadata.CONTENT_TYPE, ctype.getBaseType().toLowerCase(Locale.ENGLISH));
- md.set(Metadata.RESOURCE_NAME_KEY, fileName);
+ md.set(HttpHeaders.CONTENT_TYPE, ctype.getBaseType().toLowerCase(Locale.ENGLISH));
+ md.set(TikaMetadataKeys.RESOURCE_NAME_KEY, fileName);
String content = tika.parseToString(is, md);
if (disp != null && disp.equalsIgnoreCase(Part.ATTACHMENT)) {
if (row.get(ATTACHMENT) == null)
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java Tue Mar 6 18:13:38 2012
@@ -34,6 +34,7 @@ import org.xml.sax.helpers.DefaultHandle
import javax.xml.transform.OutputKeys;
import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
@@ -142,7 +143,7 @@ public class TikaEntityProcessor extends
private static ContentHandler getHtmlHandler(Writer writer)
throws TransformerConfigurationException {
SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
+ TransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
handler.setResult(new StreamResult(writer));
@@ -185,7 +186,7 @@ public class TikaEntityProcessor extends
private static ContentHandler getXmlContentHandler(Writer writer)
throws TransformerConfigurationException {
SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
+ TransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
handler.setResult(new StreamResult(writer));
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Tue Mar 6 18:13:38 2012
@@ -28,7 +28,6 @@ import org.apache.solr.common.util.Named
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.SystemIdResolver;
-import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.RequestHandlerBase;
@@ -39,7 +38,6 @@ import org.apache.solr.response.SolrQuer
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
-import org.apache.solr.util.SolrPluginUtils;
import org.apache.solr.util.plugin.SolrCoreAware;
import java.util.*;
@@ -109,7 +107,7 @@ public class DataImportHandler extends R
String configLoc = (String) defaults.get("config");
if (configLoc != null && configLoc.length() != 0) {
processConfiguration(defaults);
- final InputSource is = new InputSource(core.getResourceLoader().openConfig(configLoc));
+ final InputSource is = new InputSource(core.getResourceLoader().openResource(configLoc));
is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(configLoc));
importer = new DataImporter(is, core,
dataSources, coreScopeSession, myName);
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Tue Mar 6 18:13:38 2012
@@ -19,7 +19,6 @@ package org.apache.solr.handler.dataimpo
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
@@ -96,11 +95,20 @@ public class DataImporter {
*/
DataImporter() {
coreScopeSession = new ConcurrentHashMap<String, Object>();
- this.propWriter = new SimplePropertiesWriter();
+ createPropertyWriter();
propWriter.init(this);
this.handlerName = "dataimport" ;
}
+ private void createPropertyWriter() {
+ if (this.core == null
+ || !this.core.getCoreDescriptor().getCoreContainer().isZooKeeperAware()) {
+ propWriter = new SimplePropertiesWriter();
+ } else {
+ propWriter = new ZKPropertiesWriter();
+ }
+ }
+
DataImporter(InputSource dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session, String handlerName) {
this.handlerName = handlerName;
if (dataConfig == null)
@@ -108,7 +116,7 @@ public class DataImporter {
"Configuration not found");
this.core = core;
this.schema = core.getSchema();
- this.propWriter = new SimplePropertiesWriter();
+ createPropertyWriter();
propWriter.init(this);
dataSourceProps = ds;
if (session == null)
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java Tue Mar 6 18:13:38 2012
@@ -211,7 +211,7 @@ public class TestXPathEntityProcessor ex
tmpdir.delete();
tmpdir.mkdir();
tmpdir.deleteOnExit();
- TestFileListEntityProcessor.createFile(tmpdir, "x.xsl", xsl.getBytes("UTF-8"),
+ AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes("UTF-8"),
false);
Map entityAttrs = createMap("name", "e",
XPathEntityProcessor.USE_SOLR_ADD_SCHEMA, "true", "xsl", ""
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp Tue Mar 6 18:13:38 2012
@@ -47,7 +47,7 @@ Select handler:
<frameset cols = "50%, 50%">
<frame src ="debug.jsp?handler=<%=handler%>" />
- <frame src ="../select?qt=<%=handler%>&command=status" name="result"/>
+ <frame src ="..<%=handler%>?command=status" name="result"/>
</frameset>
<% } %>
</html>
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp Tue Mar 6 18:13:38 2012
@@ -30,9 +30,8 @@
<%
String handler = request.getParameter("handler"); // must be specified
%>
-<form action="../select" target="result" method="get">
+<form action="..<%=handler%>" target="result" method="get">
<input type="hidden" name="debug" value="on"/>
-<input type="hidden" name="qt" value="<%=handler%>"/>
<table>
<tr>
<th>Handler: </th>
@@ -69,17 +68,16 @@
<tr>
<td colspan="2"><textarea id="txtDataConfig" rows="30" cols="80" name="dataConfig"></textarea></td>
<script type="text/javascript" language="javascript">
- $.get("../select?qt=<%=handler%>&command=show-config", function(data){
+ $.get("..<%=handler%>?command=show-config", function(data){
$('#txtDataConfig').attr('value', data);
});
</script>
</tr>
</table>
</form>
-<form action="../select" target="result" method="get">
+<form action="..<%=handler%>" target="result" method="get">
<input type="hidden" name="clean" value="false">
<input type="hidden" name="commit" value="true">
- <input type="hidden" name="qt" value="<%=handler%>"/>
<input class="stdbutton" type="submit" name="command" value="full-import">
<input class="stdbutton" type="submit" name="command" value="delta-import">
<input class="stdbutton" type="submit" name="command" value="status">
@@ -92,8 +90,7 @@
<input type="hidden" name="rows" value="0">
<input class="stdbutton" type="submit" value="Documents Count">
</form>
-<form action="../select" target="result" method="get">
- <input type="hidden" name="qt" value="<%=handler%>"/>
+<form action="..<%=handler%>" target="result" method="get">
<input type="hidden" name="verbose" value="true">
<input type="hidden" name="clean" value="true">
<input type="hidden" name="commit" value="true">
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue Mar 6 18:13:38 2012
@@ -36,7 +36,9 @@ import org.apache.solr.update.AddUpdateC
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.DefaultParser;
@@ -150,11 +152,11 @@ public class ExtractingDocumentLoader ex
// then Tika can make use of it in guessing the appropriate MIME type:
String resourceName = req.getParams().get(ExtractingParams.RESOURCE_NAME, null);
if (resourceName != null) {
- metadata.add(Metadata.RESOURCE_NAME_KEY, resourceName);
+ metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName);
}
// Provide stream's content type as hint for auto detection
if(stream.getContentType() != null) {
- metadata.add(Metadata.CONTENT_TYPE, stream.getContentType());
+ metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType());
}
InputStream inputStream = null;
@@ -167,7 +169,7 @@ public class ExtractingDocumentLoader ex
// HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadata
String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
if(charset != null){
- metadata.add(Metadata.CONTENT_ENCODING, charset);
+ metadata.add(HttpHeaders.CONTENT_ENCODING, charset);
}
String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java Tue Mar 6 18:13:38 2012
@@ -24,6 +24,7 @@ import org.apache.solr.schema.DateField;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaMetadataKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
@@ -191,7 +192,7 @@ public class SolrContentHandler extends
if (sf==null && unknownFieldPrefix.length() > 0) {
name = unknownFieldPrefix + name;
sf = schema.getFieldOrNull(name);
- } else if (sf == null && defaultField.length() > 0 && name.equals(Metadata.RESOURCE_NAME_KEY) == false /*let the fall through below handle this*/){
+ } else if (sf == null && defaultField.length() > 0 && name.equals(TikaMetadataKeys.RESOURCE_NAME_KEY) == false /*let the fall through below handle this*/){
name = defaultField;
sf = schema.getFieldOrNull(name);
}
@@ -201,7 +202,7 @@ public class SolrContentHandler extends
// ExtractingDocumentLoader.load(). You shouldn't have to define a mapping for this
// field just because you specified a resource.name parameter to the handler, should
// you?
- if (sf == null && unknownFieldPrefix.length()==0 && name == Metadata.RESOURCE_NAME_KEY) {
+ if (sf == null && unknownFieldPrefix.length()==0 && name == TikaMetadataKeys.RESOURCE_NAME_KEY) {
return;
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml Tue Mar 6 18:13:38 2012
@@ -31,7 +31,7 @@
solr.RAMDirectoryFactory is memory based and not persistent. -->
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
- <luceneMatchVersion>LUCENE_40</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<requestHandler name="search" class="solr.SearchHandler" default="true">
<lst name="defaults">
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/CHANGES.txt?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/CHANGES.txt (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/CHANGES.txt Tue Mar 6 18:13:38 2012
@@ -5,9 +5,12 @@ This file describes changes to the Solr
Introduction
------------
-This module is intended to be used while indexing documents.
-Its purpose is to provide additional on the fly automatically generated fields to the Solr index.
+This module is intended to be used both as an UpdateRequestProcessor while indexing documents and as a set of tokenizer/filters
+to be configured inside the schema.xml for use during analysis phase.
+UIMAUpdateRequestProcessor purpose is to provide additional on the fly automatically generated fields to the Solr index.
Such fields could be language, concepts, keywords, sentences, named entities, etc.
+UIMA based tokenizers/filters can be used either inside plain Lucene or as index/query analyzers to be defined
+inside the schema.xml of a Solr core to create/filter tokens using specific UIMA annotations.
UIMA Dependency
---------------
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/build.xml?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/build.xml (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/build.xml Tue Mar 6 18:13:38 2012
@@ -20,9 +20,25 @@
<project name="solr-uima" default="default">
<description>
- Solr Integration with UIMA for extracting metadata from arbitrary (text) fields and enrich document with features extracted from UIMA types (language, sentences, concepts, named entities, etc.)
+ Solr Integration with UIMA for extracting metadata from arbitrary (text) fields and enrich document with features
+ extracted from UIMA types (language, sentences, concepts, named entities, etc.)
</description>
<import file="../contrib-build.xml"/>
+
+ <path id="classpath">
+ <pathelement path="${analyzers-uima.jar}"/>
+ <path refid="solr.base.classpath"/>
+ </path>
+
+ <target name="module-jars-to-solr" depends="jar-analyzers-uima">
+ <mkdir dir="${build.dir}/lucene-libs"/>
+ <copy todir="${build.dir}/lucene-libs" preservelastmodified="true" flatten="true" failonerror="true" overwrite="true">
+ <fileset file="${analyzers-uima.jar}"/>
+ </copy>
+ </target>
+
+ <target name="compile-core" depends="jar-analyzers-uima, solr-contrib-build.compile-core"/>
+ <target name="dist" depends="module-jars-to-solr, common-solr.dist"/>
</project>
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java Tue Mar 6 18:13:38 2012
@@ -23,14 +23,16 @@ import org.apache.solr.common.SolrInputD
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
-import org.apache.solr.uima.processor.ae.AEProvider;
-import org.apache.solr.uima.processor.ae.AEProviderFactory;
+import org.apache.lucene.analysis.uima.ae.AEProvider;
+import org.apache.lucene.analysis.uima.ae.AEProviderFactory;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
@@ -42,6 +44,8 @@ import java.util.Map;
*/
public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor {
+ private final Logger log = LoggerFactory.getLogger(UIMAUpdateRequestProcessor.class);
+
SolrUIMAConfiguration solrUIMAConfiguration;
private AEProvider aeProvider;
Modified: lucene/dev/branches/solr_3159_jetty8/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Tue Mar 6 18:13:38 2012
@@ -25,6 +25,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.velocity.Template;
import org.apache.velocity.VelocityContext;
import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.runtime.RuntimeConstants;
import org.apache.velocity.tools.generic.*;
import java.io.*;
@@ -117,14 +118,14 @@ public class VelocityResponseWriter impl
if (template_root != null) {
baseDir = new File(template_root);
}
- engine.setProperty(VelocityEngine.FILE_RESOURCE_LOADER_PATH, baseDir.getAbsolutePath());
+ engine.setProperty(RuntimeConstants.FILE_RESOURCE_LOADER_PATH, baseDir.getAbsolutePath());
engine.setProperty("params.resource.loader.instance", new SolrParamResourceLoader(request));
SolrVelocityResourceLoader resourceLoader =
new SolrVelocityResourceLoader(request.getCore().getSolrConfig().getResourceLoader());
engine.setProperty("solr.resource.loader.instance", resourceLoader);
// TODO: Externalize Velocity properties
- engine.setProperty(VelocityEngine.RESOURCE_LOADER, "params,file,solr");
+ engine.setProperty(RuntimeConstants.RESOURCE_LOADER, "params,file,solr");
String propFile = request.getParams().get("v.properties");
try {
if (propFile == null)
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java Tue Mar 6 18:13:38 2012
@@ -28,8 +28,7 @@ import java.util.Map;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer;
-import org.apache.lucene.analysis.kuromoji.Segmenter;
-import org.apache.lucene.analysis.kuromoji.Segmenter.Mode;
+import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Mode;
import org.apache.lucene.analysis.kuromoji.dict.UserDictionary;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.analysis.BaseTokenizerFactory;
@@ -88,7 +87,7 @@ public class KuromojiTokenizerFactory ex
@Override
public Tokenizer create(Reader input) {
- return new KuromojiTokenizer(new Segmenter(userDictionary, mode), input);
+ return new KuromojiTokenizer(input, userDictionary, true, mode);
}
private Mode getMode(Map<String, String> args) {
@@ -96,7 +95,7 @@ public class KuromojiTokenizerFactory ex
if (mode != null) {
return Mode.valueOf(mode.toUpperCase(Locale.ENGLISH));
} else {
- return Segmenter.DEFAULT_MODE;
+ return KuromojiTokenizer.DEFAULT_MODE;
}
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Tue Mar 6 18:13:38 2012
@@ -59,7 +59,7 @@ public abstract class ElectionContext {
// the given core may or may not be null - if you need access to the current core, you must pass
// the core container and core name to your context impl - then use this core ref if it is not null
// else access it from the core container
- abstract void runLeaderProcess(boolean weAreReplacement, SolrCore core) throws KeeperException, InterruptedException, IOException;
+ abstract void runLeaderProcess(boolean weAreReplacement) throws KeeperException, InterruptedException, IOException;
}
class ShardLeaderElectionContextBase extends ElectionContext {
@@ -81,7 +81,7 @@ class ShardLeaderElectionContextBase ext
}
@Override
- void runLeaderProcess(boolean weAreReplacement, SolrCore core)
+ void runLeaderProcess(boolean weAreReplacement)
throws KeeperException, InterruptedException, IOException {
try {
@@ -116,7 +116,7 @@ final class ShardLeaderElectionContext e
}
@Override
- void runLeaderProcess(boolean weAreReplacement, SolrCore startupCore)
+ void runLeaderProcess(boolean weAreReplacement)
throws KeeperException, InterruptedException, IOException {
if (cc != null) {
String coreName = leaderProps.get(ZkStateReader.CORE_NAME_PROP);
@@ -124,11 +124,9 @@ final class ShardLeaderElectionContext e
try {
// the first time we are run, we will get a startupCore - after
// we will get null and must use cc.getCore
- if (startupCore == null) {
- core = cc.getCore(coreName);
- } else {
- core = startupCore;
- }
+
+ core = cc.getCore(coreName);
+
if (core == null) {
cancelElection();
throw new SolrException(ErrorCode.SERVER_ERROR, "Fatal Error, SolrCore not found:" + coreName + " in " + cc.getCoreNames());
@@ -159,14 +157,14 @@ final class ShardLeaderElectionContext e
zkController.publish(core.getCoreDescriptor(), ZkStateReader.ACTIVE);
} finally {
- if (core != null && startupCore == null) {
+ if (core != null ) {
core.close();
}
}
}
- super.runLeaderProcess(weAreReplacement, startupCore);
+ super.runLeaderProcess(weAreReplacement);
}
private void rejoinLeaderElection(String leaderSeqPath, SolrCore core)
@@ -181,7 +179,7 @@ final class ShardLeaderElectionContext e
core.getUpdateHandler().getSolrCoreState().doRecovery(cc, core.getName());
- leaderElector.joinElection(this, null); // don't pass core, pass null
+ leaderElector.joinElection(this);
}
private boolean shouldIBeLeader(ZkNodeProps leaderProps) {
@@ -249,7 +247,7 @@ final class OverseerElectionContext exte
}
@Override
- void runLeaderProcess(boolean weAreReplacement, SolrCore firstCore) throws KeeperException, InterruptedException {
+ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, InterruptedException {
final String id = leaderSeqPath.substring(leaderSeqPath.lastIndexOf("/")+1);
ZkNodeProps myProps = new ZkNodeProps("id", id);
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Tue Mar 6 18:13:38 2012
@@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory;
* Leader Election process. This class contains the logic by which a
* leader is chosen. First call * {@link #setup(ElectionContext)} to ensure
* the election process is init'd. Next call
- * {@link #joinElection(ElectionContext, SolrCore)} to start the leader election.
+ * {@link #joinElection(ElectionContext)} to start the leader election.
*
* The implementation follows the classic ZooKeeper recipe of creating an
* ephemeral, sequential node for each candidate and then looking at the set
@@ -80,13 +80,12 @@ public class LeaderElector {
* @param seq
* @param context
* @param replacement has someone else been the leader already?
- * @param core
* @throws KeeperException
* @throws InterruptedException
* @throws IOException
* @throws UnsupportedEncodingException
*/
- private void checkIfIamLeader(final int seq, final ElectionContext context, boolean replacement, SolrCore core) throws KeeperException,
+ private void checkIfIamLeader(final int seq, final ElectionContext context, boolean replacement) throws KeeperException,
InterruptedException, IOException {
// get all other numbers...
final String holdElectionPath = context.electionPath + ELECTION_NODE;
@@ -95,7 +94,7 @@ public class LeaderElector {
sortSeqs(seqs);
List<Integer> intSeqs = getSeqs(seqs);
if (seq <= intSeqs.get(0)) {
- runIamLeaderProcess(context, replacement, core);
+ runIamLeaderProcess(context, replacement);
} else {
// I am not the leader - watch the node below me
int i = 1;
@@ -119,7 +118,7 @@ public class LeaderElector {
public void process(WatchedEvent event) {
// am I the next leader?
try {
- checkIfIamLeader(seq, context, true, null);
+ checkIfIamLeader(seq, context, true);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
@@ -137,15 +136,15 @@ public class LeaderElector {
} catch (KeeperException e) {
// we couldn't set our watch - the node before us may already be down?
// we need to check if we are the leader again
- checkIfIamLeader(seq, context, true, null);
+ checkIfIamLeader(seq, context, true);
}
}
}
// TODO: get this core param out of here
- protected void runIamLeaderProcess(final ElectionContext context, boolean weAreReplacement, SolrCore core) throws KeeperException,
+ protected void runIamLeaderProcess(final ElectionContext context, boolean weAreReplacement) throws KeeperException,
InterruptedException, IOException {
- context.runLeaderProcess(weAreReplacement, core);
+ context.runLeaderProcess(weAreReplacement);
}
/**
@@ -206,7 +205,7 @@ public class LeaderElector {
* @throws IOException
* @throws UnsupportedEncodingException
*/
- public int joinElection(ElectionContext context, SolrCore core) throws KeeperException, InterruptedException, IOException {
+ public int joinElection(ElectionContext context) throws KeeperException, InterruptedException, IOException {
final String shardsElectZkPath = context.electionPath + LeaderElector.ELECTION_NODE;
long sessionId = zkClient.getSolrZooKeeper().getSessionId();
@@ -249,7 +248,7 @@ public class LeaderElector {
}
}
int seq = getSeq(leaderSeqPath);
- checkIfIamLeader(seq, context, false, core);
+ checkIfIamLeader(seq, context, false);
return seq;
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/Overseer.java Tue Mar 6 18:13:38 2012
@@ -171,7 +171,7 @@ public class Overseer implements NodeSta
private boolean amILeader() {
try {
- ZkNodeProps props = ZkNodeProps.load(zkClient.getData("/overseer_elect/leader", null, null, false));
+ ZkNodeProps props = ZkNodeProps.load(zkClient.getData("/overseer_elect/leader", null, null, true));
if(myId.equals(props.get("id"))) {
return true;
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java Tue Mar 6 18:13:38 2012
@@ -45,6 +45,8 @@ import org.apache.solr.handler.Replicati
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.PeerSync;
import org.apache.solr.update.UpdateLog;
@@ -186,29 +188,44 @@ public class RecoveryStrategy extends Th
server.request(prepCmd);
server.shutdown();
}
-
+
@Override
public void run() {
- boolean replayed = false;
- boolean succesfulRecovery = false;
-
SolrCore core = cc.getCore(coreName);
if (core == null) {
SolrException.log(log, "SolrCore not found - cannot recover:" + coreName);
return;
}
- UpdateLog ulog;
+
+ // set request info for logging
try {
- ulog = core.getUpdateHandler().getUpdateLog();
- if (ulog == null) {
- SolrException.log(log, "No UpdateLog found - cannot recover");
- recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
- core.getCoreDescriptor());
- return;
- }
+ SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
+
+ log.info("Starting recovery process. recoveringAfterStartup=" + recoveringAfterStartup);
+
+ doRecovery(core);
} finally {
- core.close();
+ if (core != null) core.close();
+ SolrRequestInfo.clearRequestInfo();
}
+ }
+
+ // TODO: perhaps make this grab a new core each time through the loop to handle core reloads?
+ public void doRecovery(SolrCore core) {
+ boolean replayed = false;
+ boolean successfulRecovery = false;
+
+ UpdateLog ulog;
+ ulog = core.getUpdateHandler().getUpdateLog();
+ if (ulog == null) {
+ SolrException.log(log, "No UpdateLog found - cannot recover");
+ recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
+ core.getCoreDescriptor());
+ return;
+ }
+
List<Long> startingRecentVersions;
UpdateLog.RecentUpdates startingRecentUpdates = ulog.getRecentUpdates();
@@ -229,14 +246,14 @@ public class RecoveryStrategy extends Th
if (startingRecentVersions.get(oldIdx) == firstStartingVersion) break;
}
- if (oldIdx < startingRecentVersions.size()) {
- log.info("####### Found new versions added after startup: num=" + (startingRecentVersions.size()-oldIdx));
+ if (oldIdx > 0) {
+ log.info("####### Found new versions added after startup: num=" + oldIdx);
+ log.info("###### currentVersions=" + startingRecentVersions);
}
-
+
log.info("###### startupVersions=" + reallyStartingVersions);
- log.info("###### currentVersions=" + startingRecentVersions);
}
-
+
if (recoveringAfterStartup) {
// if we're recovering after startup (i.e. we have been down), then we need to know what the last versions were
// when we went down.
@@ -245,29 +262,24 @@ public class RecoveryStrategy extends Th
boolean firstTime = true;
- while (!succesfulRecovery && !close && !isInterrupted()) { // don't use interruption or it will close channels though
- core = cc.getCore(coreName);
- if (core == null) {
- SolrException.log(log, "SolrCore not found - cannot recover:" + coreName);
- return;
- }
+ while (!successfulRecovery && !close && !isInterrupted()) { // don't use interruption or it will close channels though
try {
// first thing we just try to sync
zkController.publish(core.getCoreDescriptor(), ZkStateReader.RECOVERING);
-
+
CloudDescriptor cloudDesc = core.getCoreDescriptor()
.getCloudDescriptor();
ZkNodeProps leaderprops = zkStateReader.getLeaderProps(
cloudDesc.getCollectionName(), cloudDesc.getShardId());
-
+
String leaderBaseUrl = leaderprops.get(ZkStateReader.BASE_URL_PROP);
String leaderCoreName = leaderprops.get(ZkStateReader.CORE_NAME_PROP);
-
- String leaderUrl = ZkCoreNodeProps.getCoreUrl(leaderBaseUrl, leaderCoreName);
-
+
+ String leaderUrl = ZkCoreNodeProps.getCoreUrl(leaderBaseUrl, leaderCoreName);
+
sendPrepRecoveryCmd(leaderBaseUrl, leaderCoreName);
-
-
+
+
// first thing we just try to sync
if (firstTime) {
firstTime = false; // only try sync the first time through the loop
@@ -282,11 +294,11 @@ public class RecoveryStrategy extends Th
SolrQueryRequest req = new LocalSolrQueryRequest(core,
new ModifiableSolrParams());
core.getUpdateHandler().commit(new CommitUpdateCommand(req, false));
- log.info("Sync Recovery was succesful - registering as Active");
+ log.info("Sync Recovery was successful - registering as Active");
// System.out
- // .println("Sync Recovery was succesful - registering as Active "
+ // .println("Sync Recovery was successful - registering as Active "
// + zkController.getNodeName());
-
+
// solrcloud_debug
// try {
// RefCounted<SolrIndexSearcher> searchHolder =
@@ -302,43 +314,43 @@ public class RecoveryStrategy extends Th
// } catch (Exception e) {
//
// }
-
+
// sync success - register as active and return
zkController.publishAsActive(baseUrl, core.getCoreDescriptor(),
coreZkNodeName, coreName);
- succesfulRecovery = true;
+ successfulRecovery = true;
close = true;
return;
}
-
+
log.info("Sync Recovery was not successful - trying replication");
}
//System.out.println("Sync Recovery was not successful - trying replication");
-
+
log.info("Begin buffering updates");
ulog.bufferUpdates();
replayed = false;
-
+
try {
-
+
replicate(zkController.getNodeName(), core,
leaderprops, leaderUrl);
-
+
replay(ulog);
replayed = true;
-
- log.info("Recovery was succesful - registering as Active");
+
+ log.info("Recovery was successful - registering as Active");
// if there are pending recovery requests, don't advert as active
zkController.publishAsActive(baseUrl, core.getCoreDescriptor(),
coreZkNodeName, coreName);
close = true;
- succesfulRecovery = true;
+ successfulRecovery = true;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("Recovery was interrupted", e);
retries = INTERRUPTED;
} catch (Throwable t) {
- SolrException.log(log, "Error while trying to recover", t);
+ log.error("Error while trying to recover", t);
} finally {
if (!replayed) {
try {
@@ -349,45 +361,34 @@ public class RecoveryStrategy extends Th
}
}
-
+
} catch (Throwable t) {
- SolrException.log(log, "Error while trying to recover", t);
- } finally {
- if (core != null) {
- core.close();
- }
+ log.error("Error while trying to recover.", t);
}
-
- if (!succesfulRecovery) {
+
+ if (!successfulRecovery) {
// lets pause for a moment and we need to try again...
// TODO: we don't want to retry for some problems?
// Or do a fall off retry...
try {
-
- SolrException.log(log, "Recovery failed - trying again...");
+
+ log.error("Recovery failed - trying again...");
retries++;
if (retries >= MAX_RETRIES) {
if (retries == INTERRUPTED) {
-
+
} else {
- // TODO: for now, give up after X tries - should we do more?
- core = cc.getCore(coreName);
- try {
- recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
- core.getCoreDescriptor());
- } finally {
- if (core != null) {
- core.close();
- }
- }
+ log.error("Recovery failed - max retries exceeded.");
+ recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
+ core.getCoreDescriptor());
}
break;
}
-
+
} catch (Exception e) {
SolrException.log(log, "", e);
}
-
+
try {
Thread.sleep(Math.min(START_TIMEOUT * retries, 60000));
} catch (InterruptedException e) {
@@ -396,11 +397,10 @@ public class RecoveryStrategy extends Th
retries = INTERRUPTED;
}
}
-
-
- log.info("Finished recovery process");
-
+
}
+ log.info("Finished recovery process");
+
}
private Future<RecoveryInfo> replay(UpdateLog ulog)
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java Tue Mar 6 18:13:38 2012
@@ -45,17 +45,21 @@ public class SolrZkServer {
String zkRun;
String zkHost;
- String solrHome;
+
String solrPort;
Properties props;
SolrZkServerProps zkProps;
private Thread zkThread; // the thread running a zookeeper server, only if zkRun is set
- public SolrZkServer(String zkRun, String zkHost, String solrHome, String solrPort) {
+ private String dataHome;
+ private String confHome;
+
+ public SolrZkServer(String zkRun, String zkHost, String dataHome, String confHome, String solrPort) {
this.zkRun = zkRun;
this.zkHost = zkHost;
- this.solrHome = solrHome;
+ this.dataHome = dataHome;
+ this.confHome = confHome;
this.solrPort = solrPort;
}
@@ -74,13 +78,13 @@ public class SolrZkServer {
zkProps = new SolrZkServerProps();
// set default data dir
// TODO: use something based on IP+port??? support ensemble all from same solr home?
- zkProps.setDataDir(solrHome + '/' + "zoo_data");
+ zkProps.setDataDir(dataHome);
zkProps.zkRun = zkRun;
zkProps.solrPort = solrPort;
}
try {
- props = SolrZkServerProps.getProperties(solrHome + '/' + "zoo.cfg");
+ props = SolrZkServerProps.getProperties(confHome + '/' + "zoo.cfg");
SolrZkServerProps.injectServers(props, zkRun, zkHost);
zkProps.parseProperties(props);
if (zkProps.getClientPortAddress() == null) {