You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by ma...@apache.org on 2010/02/11 16:14:45 UTC
svn commit: r909016 - in /lucene/solr/branches/cloud: ./
contrib/dataimporthandler/
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/
contrib/extracti...
Author: markrmiller
Date: Thu Feb 11 15:14:38 2010
New Revision: 909016
URL: http://svn.apache.org/viewvc?rev=909016&view=rev
Log:
merge up to r908357
Modified:
lucene/solr/branches/cloud/ (props changed)
lucene/solr/branches/cloud/CHANGES.txt
lucene/solr/branches/cloud/contrib/dataimporthandler/CHANGES.txt
lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ThreadedEntityProcessorWrapper.java
lucene/solr/branches/cloud/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
lucene/solr/branches/cloud/contrib/extraction/CHANGES.txt
lucene/solr/branches/cloud/contrib/extraction/src/main/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
lucene/solr/branches/cloud/src/common/org/apache/solr/common/params/HighlightParams.java
lucene/solr/branches/cloud/src/common/org/apache/solr/common/util/FileUtils.java
lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrCore.java
lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrEventListener.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/DumpRequestHandler.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/ReplicationHandler.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/SnapPuller.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/StatsValues.java
lucene/solr/branches/cloud/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/solr/branches/cloud/src/java/org/apache/solr/response/RawResponseWriter.java
lucene/solr/branches/cloud/src/java/org/apache/solr/update/UpdateHandler.java
lucene/solr/branches/cloud/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java
lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/HighlighterTest.java
Propchange: lucene/solr/branches/cloud/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Feb 11 15:14:38 2010
@@ -1 +1 @@
-/lucene/solr/trunk:888480-906068
+/lucene/solr/trunk:888480-908357
Modified: lucene/solr/branches/cloud/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/CHANGES.txt?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/CHANGES.txt (original)
+++ lucene/solr/branches/cloud/CHANGES.txt Thu Feb 11 15:14:38 2010
@@ -70,7 +70,7 @@
* SOLR-1177: Distributed Search support for TermsComponent (Matt Weber via shalin)
-* SOLR-1621: Allow current single core deployments to be specified by solr.xml (Mark Miller , noble)
+* SOLR-1621, SOLR-1722: Allow current single core deployments to be specified by solr.xml (Mark Miller , noble)
* SOLR-1532: Allow StreamingUpdateSolrServer to use a provided HttpClient (Gabriele Renzi via shalin)
@@ -164,6 +164,16 @@
* SOLR-1711: SolrJ - StreamingUpdateSolrServer had a race condition that
could halt the streaming of documents. (Attila Babo via yonik)
+
+* SOLR-1748, SOLR-1747, SOLR-1746, SOLR-1745, SOLR-1744: Streams and Readers
+ retrieved from ContentStreams are not closed in various places, resulting
+ in file descriptor leaks.
+ (Christoff Brill, Mark Miller)
+
+* SOLR-1753: StatsComponent throws NPE when getting statistics for facets in distributed search
+ (Janne Majaranta via koji)
+
+* SOLR-1736:In the slave , If 'mov'ing file does not succeed , copy the file (noble)
Other Changes
----------------------
@@ -218,6 +228,8 @@
* SOLR-1696 : Deprecate old <highlighting> syntax and move configuration to HighlightComponent (noble)
* Upgraded to Lucene 2.9-dev r900226 (koji)
+
+* SOLR-1727: SolrEventListener should extend NamedListInitializedPlugin (noble)
Build
----------------------
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/CHANGES.txt?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/CHANGES.txt Thu Feb 11 15:14:38 2010
@@ -53,6 +53,16 @@
* SOLR-1598: Reader used in PlainTextEntityProcessor is not explicitly closed (Sascha Szott via noble)
+* SOLR-1759: $skipDoc was not working correctly (Gian Marco Tagliani via noble)
+
+* SOLR-1762: DateFormatTransformer does not work correctly with non-default locale dates (tommy chheng via noble)
+
+* SOLR-1757: DIH multithreading sometimes throws NPE (noble)
+
+* SOLR-1766: DIH with threads enabled doesn't respond to the abort command (Michael Henson via noble)
+
+* SOLR-1767: dataimporter.functions.escapeSql() does not escape backslash character (Sean Timm via noble)
+
Other Changes
----------------------
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java Thu Feb 11 15:14:38 2010
@@ -47,7 +47,14 @@
@SuppressWarnings("unchecked")
public Object transformRow(Map<String, Object> aRow, Context context) {
+
for (Map<String, String> map : context.getAllEntityFields()) {
+ Locale locale = Locale.getDefault();
+ String customLocale = map.get("locale");
+ if(customLocale != null){
+ locale = new Locale(customLocale);
+ }
+
String fmt = map.get(DATE_TIME_FMT);
if (fmt == null)
continue;
@@ -61,12 +68,12 @@
List inputs = (List) o;
List<Date> results = new ArrayList<Date>();
for (Object input : inputs) {
- results.add(process(input, fmt));
+ results.add(process(input, fmt, locale));
}
aRow.put(column, results);
} else {
if (o != null) {
- aRow.put(column, process(o, fmt));
+ aRow.put(column, process(o, fmt, locale));
}
}
} catch (ParseException e) {
@@ -76,14 +83,14 @@
return aRow;
}
- private Date process(Object value, String format) throws ParseException {
+ private Date process(Object value, String format, Locale locale) throws ParseException {
if (value == null) return null;
String strVal = value.toString().trim();
if (strVal.length() == 0)
return null;
SimpleDateFormat fmt = fmtCache.get(format);
if (fmt == null) {
- fmt = new SimpleDateFormat(format);
+ fmt = new SimpleDateFormat(format, locale);
fmtCache.put(format, fmt);
}
return fmt.parse(strVal);
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Thu Feb 11 15:14:38 2010
@@ -345,11 +345,11 @@
private DocWrapper docWrapper;
private volatile boolean entityInitialized ;
String currentProcess;
- ThreadLocal<ThreadedEntityProcessorWrapper> currentEntityProcWrapper = new ThreadLocal<ThreadedEntityProcessorWrapper>();
+ final ThreadLocal<ThreadedEntityProcessorWrapper> currentEntityProcWrapper = new ThreadLocal<ThreadedEntityProcessorWrapper>();
private ContextImpl context;
- EntityRunner parent;
- AtomicBoolean entityEnded = new AtomicBoolean(false);
+ final EntityRunner parent;
+ final AtomicBoolean entityEnded = new AtomicBoolean(false);
private Exception exception;
public EntityRunner(DataConfig.Entity entity, EntityRunner parent) {
@@ -429,6 +429,7 @@
DocWrapper docWrapper = this.docWrapper;
Context.CURRENT_CONTEXT.set(context);
for (; ;) {
+ if(DocBuilder.this.stop.get()) break;
try {
Map<String, Object> arow = epw.nextRow();
if (arow == null) {
@@ -635,6 +636,7 @@
if (isRoot) {
if (e.getErrCode() == DataImportHandlerException.SKIP) {
importStatistics.skipDocCount.getAndIncrement();
+ doc = null;
} else {
LOG.error("Exception while processing: "
+ entity.name + " document : " + doc, e);
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java Thu Feb 11 15:14:38 2010
@@ -72,7 +72,10 @@
throw new DataImportHandlerException(SEVERE, "'escapeSql' must have at least one parameter ");
}
String s = l.get(0).toString();
- return s.replaceAll("'", "''").replaceAll("\"", "\"\"");
+ // escape single quote with two single quotes, double quote
+ // with two doule quotes, and backslash with double backslash.
+ // See: http://dev.mysql.com/doc/refman/4.1/en/mysql-real-escape-string.html
+ return s.replaceAll("'", "''").replaceAll("\"", "\"\"").replaceAll("\\\\", "\\\\\\\\");
}
};
}
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java Thu Feb 11 15:14:38 2010
@@ -126,8 +126,6 @@
return factory = new Callable<Connection>() {
public Connection call() throws Exception {
- // Resolve variables again because the variables may have changed
- resolveVariables(context, initProps);
LOG.info("Creating a connection for entity "
+ context.getEntityAttribute(DataImporter.NAME) + " with URL: "
+ url);
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ThreadedEntityProcessorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ThreadedEntityProcessorWrapper.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ThreadedEntityProcessorWrapper.java (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ThreadedEntityProcessorWrapper.java Thu Feb 11 15:14:38 2010
@@ -36,10 +36,10 @@
public class ThreadedEntityProcessorWrapper extends EntityProcessorWrapper {
private static final Logger LOG = LoggerFactory.getLogger(ThreadedEntityProcessorWrapper.class);
- DocBuilder.EntityRunner entityRunner;
+ final DocBuilder.EntityRunner entityRunner;
/**For each child entity there is one EntityRunner
*/
- Map<DataConfig.Entity ,DocBuilder.EntityRunner> children;
+ final Map<DataConfig.Entity ,DocBuilder.EntityRunner> children;
public ThreadedEntityProcessorWrapper(EntityProcessor delegate, DocBuilder docBuilder,
DocBuilder.EntityRunner entityRunner,
Modified: lucene/solr/branches/cloud/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java (original)
+++ lucene/solr/branches/cloud/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java Thu Feb 11 15:14:38 2010
@@ -49,6 +49,7 @@
sqlTests = new HashMap<String, String>();
sqlTests.put("foo\"", "foo\"\"");
+ sqlTests.put("foo\\", "foo\\\\");
sqlTests.put("foo'", "foo''");
sqlTests.put("foo''", "foo''''");
sqlTests.put("'foo\"", "''foo\"\"");
Modified: lucene/solr/branches/cloud/contrib/extraction/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/extraction/CHANGES.txt?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/extraction/CHANGES.txt (original)
+++ lucene/solr/branches/cloud/contrib/extraction/CHANGES.txt Thu Feb 11 15:14:38 2010
@@ -23,6 +23,9 @@
* SOLR-1567: Upgrade to Tika 0.5, which upgrades many of the underlying libraries (PDFBox, for example) too (gsingers)
+* SOLR-1756: The date.format setting causes ClassCastException when enabled and the config code that
+ parses this setting does not properly use the same iterator instance. (Christoph Brill, Mark Miller)
+
================== Release 1.4.0 ==================
1. SOLR-284: Added in support for extraction. (Eric Pugh, Chris Harris, gsingers)
Modified: lucene/solr/branches/cloud/contrib/extraction/src/main/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/contrib/extraction/src/main/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/contrib/extraction/src/main/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java (original)
+++ lucene/solr/branches/cloud/contrib/extraction/src/main/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java Thu Feb 11 15:14:38 2010
@@ -35,6 +35,8 @@
import java.io.File;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
/**
@@ -81,8 +83,9 @@
NamedList configDateFormats = (NamedList) initArgs.get(DATE_FORMATS);
if (configDateFormats != null && configDateFormats.size() > 0) {
dateFormats = new HashSet<String>();
- while (configDateFormats.iterator().hasNext()) {
- String format = (String) configDateFormats.iterator().next();
+ Iterator<Map.Entry> it = configDateFormats.iterator();
+ while (it.hasNext()) {
+ String format = (String) it.next().getValue();
log.info("Adding Date Format: " + format);
dateFormats.add(format);
}
Modified: lucene/solr/branches/cloud/src/common/org/apache/solr/common/params/HighlightParams.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/common/org/apache/solr/common/params/HighlightParams.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/common/org/apache/solr/common/params/HighlightParams.java (original)
+++ lucene/solr/branches/cloud/src/common/org/apache/solr/common/params/HighlightParams.java Thu Feb 11 15:14:38 2010
@@ -41,7 +41,7 @@
public static final String MERGE_CONTIGUOUS_FRAGMENTS = HIGHLIGHT + ".mergeContiguous";
- public static final String USE_HIGHLIGHTER = HIGHLIGHT + ".useHighlighter";
+ public static final String USE_FVH = HIGHLIGHT + ".useFastVectorHighlighter";
// Formatter
public static final String SIMPLE = "simple";
Modified: lucene/solr/branches/cloud/src/common/org/apache/solr/common/util/FileUtils.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/common/org/apache/solr/common/util/FileUtils.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/common/org/apache/solr/common/util/FileUtils.java (original)
+++ lucene/solr/branches/cloud/src/common/org/apache/solr/common/util/FileUtils.java Thu Feb 11 15:14:38 2010
@@ -17,10 +17,8 @@
package org.apache.solr.common.util;
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.FileNotFoundException;
+import java.io.*;
+import java.nio.channels.FileChannel;
/**
* @version $Id$
@@ -42,6 +40,19 @@
return r.isAbsolute() ? r : new File(base, path);
}
+ public static void copyFile(File src , File destination) throws IOException {
+ FileChannel in = null;
+ FileChannel out = null;
+ try {
+ in = new FileInputStream(src).getChannel();
+ out = new FileOutputStream(destination).getChannel();
+ in.transferTo(0, in.size(), out);
+ } finally {
+ try { if (in != null) in.close(); } catch (IOException e) {}
+ try { if (out != null) out.close(); } catch (IOException e) {}
+ }
+ }
+
/**
* Copied from Lucene's {@link org.apache.lucene.store.FSDirectory#sync(String)}
*
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrCore.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrCore.java Thu Feb 11 15:14:38 2010
@@ -275,12 +275,8 @@
}
private void initListeners() {
- List<PluginInfo> l = solrConfig.getPluginInfos(SolrEventListener.class.getName());
- firstSearcherListeners = new ArrayList<SolrEventListener>();
- newSearcherListeners = new ArrayList<SolrEventListener>();
- for (PluginInfo info : l) {
- SolrEventListener listener = createEventListener(info.className);
- listener.init(info.initArgs);
+ for (PluginInfo info : solrConfig.getPluginInfos(SolrEventListener.class.getName())) {
+ SolrEventListener listener = createInitInstance(info, SolrEventListener.class,"Event Listener",null);
String event = info.attributes.get("event");
if("firstSearcher".equals(event) ){
firstSearcherListeners.add(listener);
@@ -291,8 +287,8 @@
}
}
- List<SolrEventListener> firstSearcherListeners;
- List<SolrEventListener> newSearcherListeners;
+ final List<SolrEventListener> firstSearcherListeners = new ArrayList<SolrEventListener>();
+ final List<SolrEventListener> newSearcherListeners = new ArrayList<SolrEventListener>();
/**
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrEventListener.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrEventListener.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrEventListener.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/SolrEventListener.java Thu Feb 11 15:14:38 2010
@@ -19,6 +19,7 @@
import org.apache.solr.common.util.NamedList;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,10 +27,9 @@
/**
* @version $Id$
*/
-public interface SolrEventListener {
+public interface SolrEventListener extends NamedListInitializedPlugin{
static final Logger log = LoggerFactory.getLogger(SolrCore.class);
- public void init(NamedList args);
public void postCommit();
@@ -61,4 +61,4 @@
*/
public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher);
-}
\ No newline at end of file
+}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java Thu Feb 11 15:14:38 2010
@@ -49,7 +49,15 @@
protected ContentStreamLoader newLoader(SolrQueryRequest req, final UpdateRequestProcessor processor) {
return new ContentStreamLoader() {
public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws Exception {
- parseAndLoadDocs(req, rsp, stream.getStream(), processor);
+ InputStream is = null;
+ try {
+ is = stream.getStream();
+ parseAndLoadDocs(req, rsp, is, processor);
+ } finally {
+ if(is != null) {
+ is.close();
+ }
+ }
}
};
}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/DumpRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/DumpRequestHandler.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/DumpRequestHandler.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/DumpRequestHandler.java Thu Feb 11 15:14:38 2010
@@ -18,6 +18,7 @@
package org.apache.solr.handler;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import org.apache.commons.io.IOUtils;
@@ -45,7 +46,12 @@
stream.add( "sourceInfo", content.getSourceInfo() );
stream.add( "size", content.getSize() );
stream.add( "contentType", content.getContentType() );
- stream.add( "stream", IOUtils.toString( content.getStream() ) );
+ InputStream is = content.getStream();
+ try {
+ stream.add( "stream", IOUtils.toString(is) );
+ } finally {
+ is.close();
+ }
streams.add( stream );
}
rsp.add( "streams", streams );
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/MoreLikeThisHandler.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/MoreLikeThisHandler.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/MoreLikeThisHandler.java Thu Feb 11 15:14:38 2010
@@ -23,11 +23,9 @@
import java.net.URL;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.regex.Pattern;
import org.apache.lucene.document.Document;
@@ -40,7 +38,6 @@
import org.apache.lucene.search.similar.MoreLikeThis;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.DisMaxParams;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.MoreLikeThisParams;
import org.apache.solr.common.params.SolrParams;
@@ -84,24 +81,7 @@
SolrParams params = req.getParams();
SolrIndexSearcher searcher = req.getSearcher();
- // Parse Required Params
- // This will either have a single Reader or valid query
- Reader reader = null;
- String q = params.get( CommonParams.Q );
- if( q == null || q.trim().length() <1 ) {
- Iterable<ContentStream> streams = req.getContentStreams();
- if( streams != null ) {
- Iterator<ContentStream> iter = streams.iterator();
- if( iter.hasNext() ) {
- reader = iter.next().getReader();
- }
- if( iter.hasNext() ) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
- "MoreLikeThis does not support multiple ContentStreams" );
- }
- }
- }
-
+
MoreLikeThisHelper mlt = new MoreLikeThisHelper( params, searcher );
List<Query> filters = SolrPluginUtils.parseFilterQueries(req);
@@ -110,46 +90,75 @@
List<InterestingTerm> interesting = (termStyle == TermStyle.NONE )
? null : new ArrayList<InterestingTerm>( mlt.mlt.getMaxQueryTerms() );
- // What fields do we need to return
- String fl = params.get(CommonParams.FL);
- int flags = 0;
- if (fl != null) {
- flags |= SolrPluginUtils.setReturnFields(fl, rsp);
- }
-
- int start = params.getInt( CommonParams.START, 0 );
- int rows = params.getInt( CommonParams.ROWS, 10 );
-
DocListAndSet mltDocs = null;
+ String q = params.get( CommonParams.Q );
- // Find documents MoreLikeThis - either with a reader or a query
- //--------------------------------------------------------------------------------
- if( reader != null ) {
- mltDocs = mlt.getMoreLikeThis( reader, start, rows, filters, interesting, flags );
- }
- else if( q != null ) {
- // Matching options
- boolean includeMatch = params.getBool( MoreLikeThisParams.MATCH_INCLUDE, true );
- int matchOffset = params.getInt( MoreLikeThisParams.MATCH_OFFSET, 0 );
- // Find the base match
- Query query = QueryParsing.parseQuery(q, params.get(CommonParams.DF), params, req.getSchema());
- DocList match = searcher.getDocList(query, null, null, matchOffset, 1, flags ); // only get the first one...
- if( includeMatch ) {
- rsp.add( "match", match );
- }
-
- // This is an iterator, but we only handle the first match
- DocIterator iterator = match.iterator();
- if( iterator.hasNext() ) {
- // do a MoreLikeThis query for each document in results
- int id = iterator.nextDoc();
- mltDocs = mlt.getMoreLikeThis( id, start, rows, filters, interesting, flags );
+ // Parse Required Params
+ // This will either have a single Reader or valid query
+ Reader reader = null;
+ try {
+ if (q == null || q.trim().length() < 1) {
+ Iterable<ContentStream> streams = req.getContentStreams();
+ if (streams != null) {
+ Iterator<ContentStream> iter = streams.iterator();
+ if (iter.hasNext()) {
+ reader = iter.next().getReader();
+ }
+ if (iter.hasNext()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "MoreLikeThis does not support multiple ContentStreams");
+ }
+ }
+ }
+
+ // What fields do we need to return
+ String fl = params.get(CommonParams.FL);
+ int flags = 0;
+ if (fl != null) {
+ flags |= SolrPluginUtils.setReturnFields(fl, rsp);
+ }
+
+ int start = params.getInt(CommonParams.START, 0);
+ int rows = params.getInt(CommonParams.ROWS, 10);
+
+ // Find documents MoreLikeThis - either with a reader or a query
+ // --------------------------------------------------------------------------------
+ if (reader != null) {
+ mltDocs = mlt.getMoreLikeThis(reader, start, rows, filters,
+ interesting, flags);
+ } else if (q != null) {
+ // Matching options
+ boolean includeMatch = params.getBool(MoreLikeThisParams.MATCH_INCLUDE,
+ true);
+ int matchOffset = params.getInt(MoreLikeThisParams.MATCH_OFFSET, 0);
+ // Find the base match
+ Query query = QueryParsing.parseQuery(q, params.get(CommonParams.DF),
+ params, req.getSchema());
+ DocList match = searcher.getDocList(query, null, null, matchOffset, 1,
+ flags); // only get the first one...
+ if (includeMatch) {
+ rsp.add("match", match);
+ }
+
+ // This is an iterator, but we only handle the first match
+ DocIterator iterator = match.iterator();
+ if (iterator.hasNext()) {
+ // do a MoreLikeThis query for each document in results
+ int id = iterator.nextDoc();
+ mltDocs = mlt.getMoreLikeThis(id, start, rows, filters, interesting,
+ flags);
+ }
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "MoreLikeThis requires either a query (?q=) or text to find similar documents.");
+ }
+
+ } finally {
+ if (reader != null) {
+ reader.close();
}
}
- else {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
- "MoreLikeThis requires either a query (?q=) or text to find similar documents." );
- }
+
if( mltDocs == null ) {
mltDocs = new DocListAndSet(); // avoid NPE
}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/ReplicationHandler.java Thu Feb 11 15:14:38 2010
@@ -279,14 +279,15 @@
try {
IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
IndexCommit indexCommit = delPolicy.getLatestCommit();
- // race?
- delPolicy.setReserveDuration(indexCommit.getVersion(), reserveCommitDuration);
+
if(indexCommit == null) {
indexCommit = req.getSearcher().getReader().getIndexCommit();
+ // race?
+ delPolicy.setReserveDuration(indexCommit.getVersion(), reserveCommitDuration);
}
- if (indexCommit != null) {
- new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, this);
- }
+
+ new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, this);
+
} catch (Exception e) {
LOG.warn("Exception during creating a snapshot", e);
rsp.add("exception", e);
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/SnapPuller.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/SnapPuller.java Thu Feb 11 15:14:38 2010
@@ -569,15 +569,23 @@
/**
* Copy a file by the File#renameTo() method. If it fails, it is considered a failure
* <p/>
- * Todo may be we should try a simple copy if it fails
*/
private boolean copyAFile(File tmpIdxDir, File indexDir, String fname, List<String> copiedfiles) {
File indexFileInTmpDir = new File(tmpIdxDir, fname);
File indexFileInIndex = new File(indexDir, fname);
boolean success = indexFileInTmpDir.renameTo(indexFileInIndex);
+ if(!success){
+ try {
+ LOG.error("Unable to move index file from: " + indexFileInTmpDir
+ + " to: " + indexFileInIndex + "Trying to do a copy");
+ FileUtils.copyFile(indexFileInTmpDir,indexFileInIndex);
+ success = true;
+ } catch (IOException e) {
+ LOG.error("Unable to copy index file from: " + indexFileInTmpDir
+ + " to: " + indexFileInIndex , e);
+ }
+ }
if (!success) {
- LOG.error("Unable to move index file from: " + indexFileInTmpDir
- + " to: " + indexFileInIndex);
for (String f : copiedfiles) {
File indexFile = new File(indexDir, f);
if (indexFile.exists())
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/StatsValues.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/StatsValues.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/StatsValues.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/StatsValues.java Thu Feb 11 15:14:38 2010
@@ -73,13 +73,13 @@
facets.put( field, addTo );
}
for( int j=0; j< vals.size(); j++ ) {
- String val = f.getName(i);
+ String val = vals.getName(j);
StatsValues vvals = addTo.get( val );
if( vvals == null ) {
vvals = new StatsValues();
addTo.put( val, vvals );
}
- vvals.accumulate( (NamedList)f.getVal( i ) );
+ vvals.accumulate( (NamedList)vals.getVal( j ) );
}
}
}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Thu Feb 11 15:14:38 2010
@@ -355,7 +355,7 @@
return schemaField != null &&
schemaField.storeTermPositions() &&
schemaField.storeTermOffsets() &&
- !params.getFieldBool( fieldName, HighlightParams.USE_HIGHLIGHTER, false );
+ params.getFieldBool( fieldName, HighlightParams.USE_FVH, false );
}
private void doHighlightingByHighlighter( Query query, SolrQueryRequest req, NamedList docSummaries,
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/response/RawResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/response/RawResponseWriter.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/response/RawResponseWriter.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/response/RawResponseWriter.java Thu Feb 11 15:14:38 2010
@@ -18,6 +18,7 @@
package org.apache.solr.response;
import java.io.IOException;
+import java.io.Reader;
import java.io.Writer;
import org.apache.commons.io.IOUtils;
@@ -81,7 +82,12 @@
if( obj != null && (obj instanceof ContentStream ) ) {
// copy the contents to the writer...
ContentStream content = (ContentStream)obj;
- IOUtils.copy( content.getReader(), writer );
+ Reader reader = content.getReader();
+ try {
+ IOUtils.copy( reader, writer );
+ } finally {
+ reader.close();
+ }
}
else {
getBaseWriter( request ).write( writer, request, response );
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/update/UpdateHandler.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/update/UpdateHandler.java Thu Feb 11 15:14:38 2010
@@ -61,10 +61,9 @@
private void parseEventListeners() {
for (PluginInfo pluginInfo : core.getSolrConfig().getPluginInfos(SolrEventListener.class.getName())) {
String event = pluginInfo.attributes.get("event");
- SolrEventListener listener = core.createEventListener(pluginInfo.className);
- listener.init(pluginInfo.initArgs);
+ SolrEventListener listener = core.createInitInstance(pluginInfo,SolrEventListener.class,"Event Listener",null);
if ("postCommit".equals(event)) {
- commitCallbacks.add(listener);
+ commitCallbacks.add(core.createInitInstance(pluginInfo,SolrEventListener.class,"Event Listener",null));
log.info("added SolrEventListener for postCommit: " + listener);
} else if ("postOptimize".equals(event)) {
optimizeCallbacks.add(listener);
Modified: lucene/solr/branches/cloud/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java (original)
+++ lucene/solr/branches/cloud/src/solrj/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java Thu Feb 11 15:14:38 2010
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.Reader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
@@ -246,6 +247,7 @@
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException {
HttpMethod method = null;
+ InputStream is = null;
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
@@ -333,7 +335,12 @@
@Override
protected void sendData(OutputStream out)
throws IOException {
- IOUtils.copy(c.getReader(), out);
+ Reader reader = c.getReader();
+ try {
+ IOUtils.copy(reader, out);
+ } finally {
+ reader.close();
+ }
}
});
}
@@ -378,7 +385,8 @@
);
} else {
- post.setRequestEntity(new InputStreamRequestEntity(contentStream[0].getStream(), contentStream[0].getContentType()));
+ is = contentStream[0].getStream();
+ post.setRequestEntity(new InputStreamRequestEntity(is, contentStream[0].getContentType()));
}
method = post;
}
@@ -391,6 +399,9 @@
// This is generally safe to retry on
method.releaseConnection();
method = null;
+ if(is != null) {
+ is.close();
+ }
// If out of tries then just rethrow (as normal error).
if( ( tries < 1 ) ) {
throw r;
@@ -473,6 +484,9 @@
}
finally {
method.releaseConnection();
+ if(is != null) {
+ is.close();
+ }
}
}
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/FastVectorHighlighterTest.java Thu Feb 11 15:14:38 2010
@@ -54,6 +54,7 @@
args.put("hl", "true");
args.put("hl.fl", "tv_text");
args.put("hl.snippets", "2");
+ args.put("hl.useFastVectorHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard",0,200,args);
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/HighlighterTest.java?rev=909016&r1=909015&r2=909016&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/HighlighterTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/highlight/HighlighterTest.java Thu Feb 11 15:14:38 2010
@@ -134,7 +134,6 @@
args.put("hl", "true");
args.put("hl.fl", "tv_text");
args.put("hl.snippets", "2");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard",0,200,args);
@@ -175,7 +174,6 @@
args.put("hl", "true");
args.put("hl.fl", "tv_mv_text");
args.put("hl.snippets", "2");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard",0,200,args);
@@ -202,7 +200,6 @@
args.put("hl", "true");
args.put("hl.fl", "tv_mv_text");
args.put("hl.snippets", "2");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard",0,200,args);
@@ -228,7 +225,6 @@
args.put("hl.fl", "tv_text");
args.put("qf", "tv_text");
args.put("q.alt", "*:*");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"dismax",0,200,args);
@@ -340,7 +336,6 @@
HashMap<String,String> args = new HashMap<String,String>();
args.put("hl", "true");
args.put("hl.fl", "t_text tv_text");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard", 0, 200, args);
@@ -441,7 +436,6 @@
HashMap<String,String> args = new HashMap<String,String>();
args.put("hl", "true");
args.put("hl.fl", "tv_text");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard", 0, 200, args);
@@ -540,7 +534,6 @@
HashMap<String,String> args = new HashMap<String,String>();
args.put("hl", "true");
args.put("hl.fl", "tv_text");
- args.put("hl.useHighlighter", "true");
TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
"standard", 0, 200, args);
assertQ("Basic summarization",