You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/08/23 16:07:19 UTC
svn commit: r1160700 [18/22] - in /lucene/dev/branches/flexscoring: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/
dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/contrib/...
Modified: lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java Tue Aug 23 14:06:58 2011
@@ -27,46 +27,40 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
-import java.util.Properties;
+import java.util.Map;
+import java.util.Set;
/**
- * <p> Writes documents to SOLR as well as provides methods for loading and persisting last index time. </p>
+ * <p> Writes documents to SOLR. </p>
* <p/>
* <b>This API is experimental and may change in the future.</b>
*
* @since solr 1.3
*/
-public class SolrWriter {
+public class SolrWriter implements DIHWriter {
private static final Logger log = LoggerFactory.getLogger(SolrWriter.class);
- static final String IMPORTER_PROPERTIES = "dataimport.properties";
-
static final String LAST_INDEX_KEY = "last_index_time";
private final UpdateRequestProcessor processor;
- private final String configDir;
-
- private String persistFilename = IMPORTER_PROPERTIES;
-
- DebugLogger debugLogger;
-
SolrQueryRequest req;
- public SolrWriter(UpdateRequestProcessor processor, String confDir, SolrQueryRequest req) {
+ public SolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
this.processor = processor;
- configDir = confDir;
this.req = req;
}
- public SolrWriter(UpdateRequestProcessor processor, String confDir, String filePrefix, SolrQueryRequest req) {
- this.processor = processor;
- configDir = confDir;
- if(filePrefix != null){
- persistFilename = filePrefix+".properties";
- }
- this.req = req;
+
+ @Override
+ public void close() {
+ try {
+ processor.finish();
+ } catch (IOException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to call finish() on UpdateRequestProcessor", e);
+ }
}
-
+ @Override
public boolean upload(SolrInputDocument d) {
try {
AddUpdateCommand command = new AddUpdateCommand(req);
@@ -79,7 +73,8 @@ public class SolrWriter {
return true;
}
-
+
+ @Override
public void deleteDoc(Object id) {
try {
log.info("Deleting document: " + id);
@@ -90,75 +85,8 @@ public class SolrWriter {
log.error("Exception while deleteing: " + id, e);
}
}
-
-
- void persist(Properties p) {
- OutputStream propOutput = null;
-
- Properties props = readIndexerProperties();
-
- try {
- props.putAll(p);
- File persistFile = getPersistFile();
- propOutput = new FileOutputStream(persistFile);
- props.store(propOutput, null);
- log.info("Wrote last indexed time to " + persistFile.getAbsolutePath());
- } catch (FileNotFoundException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to persist Index Start Time", e);
- } catch (IOException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to persist Index Start Time", e);
- } finally {
- try {
- if (propOutput != null)
- propOutput.close();
- } catch (IOException e) {
- propOutput = null;
- }
- }
- }
-
- File getPersistFile() {
- String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator))
- filePath += File.separator;
- filePath += persistFilename;
- return new File(filePath);
- }
-
- void finish() {
- try {
- processor.finish();
- } catch (IOException e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to call finish() on UpdateRequestProcessor", e);
- }
- }
-
- Properties readIndexerProperties() {
- Properties props = new Properties();
- InputStream propInput = null;
-
- try {
- propInput = new FileInputStream(configDir
- + persistFilename);
- props.load(propInput);
- log.info("Read " + persistFilename);
- } catch (Exception e) {
- log.warn("Unable to read: " + persistFilename);
- } finally {
- try {
- if (propInput != null)
- propInput.close();
- } catch (IOException e) {
- propInput = null;
- }
- }
-
- return props;
- }
-
+
+ @Override
public void deleteByQuery(String query) {
try {
log.info("Deleting documents from Solr with query: " + query);
@@ -170,6 +98,7 @@ public class SolrWriter {
}
}
+ @Override
public void commit(boolean optimize) {
try {
CommitUpdateCommand commit = new CommitUpdateCommand(req,optimize);
@@ -179,6 +108,7 @@ public class SolrWriter {
}
}
+ @Override
public void rollback() {
try {
RollbackUpdateCommand rollback = new RollbackUpdateCommand(req);
@@ -188,6 +118,7 @@ public class SolrWriter {
}
}
+ @Override
public void doDeleteAll() {
try {
DeleteUpdateCommand deleteCommand = new DeleteUpdateCommand(req);
@@ -225,28 +156,8 @@ public class SolrWriter {
return null;
}
}
-
- public DebugLogger getDebugLogger() {
- if (debugLogger == null) {
- debugLogger = new DebugLogger(this);
- }
- return debugLogger;
- }
-
- /**
- * This method is used for verbose debugging
- *
- * @param event The event name start.entity ,end.entity ,transformer.row
- * @param name Name of the entity/transformer
- * @param row The actual data . Can be a Map<String,object> or a List<Map<String,object>>
- */
- public void log(int event, String name, Object row) {
- getDebugLogger().log(event, name, row);
- }
-
- public static final int START_ENTITY = 1, END_ENTITY = 2,
- TRANSFORMED_ROW = 3, ENTITY_META = 4, PRE_TRANSFORMER_ROW = 5,
- START_DOC = 6, END_DOC = 7, ENTITY_OUT = 8, ROW_END = 9,
- TRANSFORMER_EXCEPTION = 10, ENTITY_EXCEPTION = 11, DISABLE_LOGGING = 12,
- ENABLE_LOGGING = 13;
+ @Override
+ public void init(Context context) {
+ /* NO-OP */
+ }
}
Modified: lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java Tue Aug 23 14:06:58 2011
@@ -35,11 +35,6 @@ import java.util.regex.Pattern;
public class VariableResolverImpl extends VariableResolver {
private Map<String, Object> container = new HashMap<String, Object>();
- /**
- * Used for creating Evaluators
- */
- Context context;
-
private final TemplateString templateString = new TemplateString();
private final Map defaults ;
Modified: lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java Tue Aug 23 14:06:58 2011
@@ -54,7 +54,7 @@ public abstract class AbstractDataImport
// note, a little twisted that we shadow this static method
public static void initCore(String config, String schema) throws Exception {
- initCore(config, schema, getFile("solr-dih").getAbsolutePath());
+ initCore(config, schema, getFile("dih/solr").getAbsolutePath());
}
@Override
Modified: lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java Tue Aug 23 14:06:58 2011
@@ -39,7 +39,7 @@ import java.util.List;
* @since solr 1.4
*/
public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCase {
- private static final String CONF_DIR = "." + File.separator + "solr-dih" + File.separator + "conf" + File.separator;
+ private static final String CONF_DIR = "dih/solr/conf/";
SolrInstance instance = null;
JettySolrRunner jetty;
Modified: lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java Tue Aug 23 14:06:58 2011
@@ -198,7 +198,7 @@ public class TestDocBuilder extends Abst
Boolean finishCalled = Boolean.FALSE;
public SolrWriterImpl() {
- super(null, ".",null);
+ super(null, null);
}
@Override
@@ -207,11 +207,6 @@ public class TestDocBuilder extends Abst
}
@Override
- public void log(int event, String name, Object row) {
- // Do nothing
- }
-
- @Override
public void doDeleteAll() {
deleteAllCalled = Boolean.TRUE;
}
@@ -222,7 +217,7 @@ public class TestDocBuilder extends Abst
}
@Override
- public void finish() {
+ public void close() {
finishCalled = Boolean.TRUE;
}
}
Modified: lucene/dev/branches/flexscoring/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue Aug 23 14:06:58 2011
@@ -119,7 +119,7 @@ public class ExtractingDocumentLoader ex
}
void addDoc(SolrContentHandler handler) throws IOException {
- templateAdd.indexedId = null;
+ templateAdd.clear();
doAdd(handler, templateAdd);
}
Modified: lucene/dev/branches/flexscoring/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Tue Aug 23 14:06:58 2011
@@ -41,7 +41,7 @@ import org.junit.Test;
public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml", "schema.xml", "solr-extraction");
+ initCore("solrconfig.xml", "schema.xml", getFile("extraction/solr").getAbsolutePath());
}
@Override
@@ -56,7 +56,7 @@ public class ExtractingRequestHandlerTes
public void testExtraction() throws Exception {
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
- loadLocal("solr-word.pdf",
+ loadLocal("extraction/solr-word.pdf",
"fmap.created", "extractedDate",
"fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
@@ -73,7 +73,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("title:solr-word"), "//*[@numFound='1']");
- loadLocal("simple.html", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/simple.html", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"fmap.language", "extractedLanguage",
@@ -86,7 +86,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("title:Welcome"), "//*[@numFound='1']");
- loadLocal("simple.html",
+ loadLocal("extraction/simple.html",
"literal.id","simple2",
"uprefix", "t_",
"lowernames", "true",
@@ -105,7 +105,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("+id:simple2 +t_abcxyz:[* TO *]"), "//*[@numFound='1']");
// load again in the exact same way, but boost one field
- loadLocal("simple.html",
+ loadLocal("extraction/simple.html",
"literal.id","simple3",
"uprefix", "t_",
"lowernames", "true",
@@ -120,7 +120,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("+id:simple3 +t_content_type:[* TO *]"), "//*[@numFound='1']");//test lowercase and then uprefix
// test capture
- loadLocal("simple.html",
+ loadLocal("extraction/simple.html",
"literal.id","simple4",
"uprefix", "t_",
"capture","p", // capture only what is in the title element
@@ -129,7 +129,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("+id:simple4 +t_content:Solr"), "//*[@numFound='1']");
assertQ(req("+id:simple4 +t_p:\"here is some text\""), "//*[@numFound='1']");
- loadLocal("version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"literal.id", "three",
@@ -152,7 +152,7 @@ public class ExtractingRequestHandlerTes
try {
ignoreException("unknown field 'a'");
ignoreException("unknown field 'meta'"); // TODO: should this exception be happening?
- loadLocal("simple.html",
+ loadLocal("extraction/simple.html",
"literal.id","simple2",
"lowernames", "true",
"captureAttr", "true",
@@ -168,7 +168,7 @@ public class ExtractingRequestHandlerTes
}
- loadLocal("simple.html",
+ loadLocal("extraction/simple.html",
"literal.id","simple2",
ExtractingParams.DEFAULT_FIELD, "defaultExtr",//test that unmapped fields go to the text field when no uprefix is specified
"lowernames", "true",
@@ -180,7 +180,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("defaultExtr:http\\://www.apache.org"), "//*[@numFound='1']");
//Test when both uprefix and default are specified.
- loadLocal("simple.html",
+ loadLocal("extraction/simple.html",
"literal.id","simple2",
ExtractingParams.DEFAULT_FIELD, "defaultExtr",//test that unmapped fields go to the text field when no uprefix is specified
ExtractingParams.UNKNOWN_FIELD_PREFIX, "t_",
@@ -198,7 +198,7 @@ public class ExtractingRequestHandlerTes
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
//test literal
- loadLocal("version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"fmap.content", "extractedContent",
@@ -217,7 +217,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("extractionLiteralMV:two"), "//*[@numFound='1']");
try {
- loadLocal("version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"fmap.content", "extractedContent",
@@ -233,7 +233,7 @@ public class ExtractingRequestHandlerTes
//nothing to see here, move along
}
- loadLocal("version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"fmap.content", "extractedContent",
@@ -253,7 +253,7 @@ public class ExtractingRequestHandlerTes
assertTrue("handler is null and it shouldn't be", handler != null);
// Load plain text specifying MIME type:
- loadLocal("version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"literal.id", "one",
@@ -272,13 +272,13 @@ public class ExtractingRequestHandlerTes
assertTrue("handler is null and it shouldn't be", handler != null);
// Load plain text specifying filename
- loadLocal("version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Author", "extractedAuthor",
"literal.id", "one",
"fmap.language", "extractedLanguage",
"fmap.content", "extractedContent",
- ExtractingParams.RESOURCE_NAME, "version_control.txt"
+ ExtractingParams.RESOURCE_NAME, "extraction/version_control.txt"
);
assertQ(req("extractedContent:Apache"), "//*[@numFound='0']");
assertU(commit());
@@ -292,7 +292,7 @@ public class ExtractingRequestHandlerTes
public void testExtractOnly() throws Exception {
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
- SolrQueryResponse rsp = loadLocal("solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true");
+ SolrQueryResponse rsp = loadLocal("extraction/solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true");
assertTrue("rsp is null and it shouldn't be", rsp != null);
NamedList list = rsp.getValues();
@@ -306,7 +306,7 @@ public class ExtractingRequestHandlerTes
assertTrue("title is null and it shouldn't be", title != null);
assertTrue(extraction.indexOf("<?xml") != -1);
- rsp = loadLocal("solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true",
+ rsp = loadLocal("extraction/solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true",
ExtractingParams.EXTRACT_FORMAT, ExtractingDocumentLoader.TEXT_FORMAT);
assertTrue("rsp is null and it shouldn't be", rsp != null);
list = rsp.getValues();
@@ -329,7 +329,7 @@ public class ExtractingRequestHandlerTes
public void testXPath() throws Exception {
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
- SolrQueryResponse rsp = loadLocal("example.html",
+ SolrQueryResponse rsp = loadLocal("extraction/example.html",
ExtractingParams.XPATH_EXPRESSION, "/xhtml:html/xhtml:body/xhtml:a/descendant:node()",
ExtractingParams.EXTRACT_ONLY, "true"
);
@@ -347,7 +347,7 @@ public class ExtractingRequestHandlerTes
h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
- loadLocal("arabic.pdf", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/arabic.pdf", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Creation-Date", "extractedDate",
"fmap.AAPL:Keywords", "ignored_a",
@@ -368,7 +368,7 @@ public class ExtractingRequestHandlerTes
assertTrue("handler is null and it shouldn't be", handler != null);
try{
- loadLocal("password-is-solrcell.docx",
+ loadLocal("extraction/password-is-solrcell.docx",
"literal.id", "one");
fail("TikaException is expected because of trying to extract text from password protected word file.");
}
@@ -377,7 +377,7 @@ public class ExtractingRequestHandlerTes
assertQ(req("*:*"), "//result[@numFound=0]");
try{
- loadLocal("password-is-solrcell.docx", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ loadLocal("extraction/password-is-solrcell.docx", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
"fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
"fmap.Creation-Date", "extractedDate",
"fmap.AAPL:Keywords", "ignored_a",
Modified: lucene/dev/branches/flexscoring/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/flexscoring/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Tue Aug 23 14:06:58 2011
@@ -50,7 +50,7 @@ public class UIMAUpdateRequestProcessorT
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml", "schema.xml", "solr-uima");
+ initCore("solrconfig.xml", "schema.xml", getFile("uima/solr").getAbsolutePath());
}
@Override
Modified: lucene/dev/branches/flexscoring/solr/core/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/build.xml?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/build.xml (original)
+++ lucene/dev/branches/flexscoring/solr/core/build.xml Tue Aug 23 14:06:58 2011
@@ -20,14 +20,7 @@
<import file="../common-build.xml"/>
- <target name="compile-core" unless="solr.core.compiled"
- depends="prep-lucene-jars, compile-solrj, common.compile-core">
- <property name="solr.core.compiled" value="true"/>
- </target>
-
- <target name="compile-test" depends="compile-solr-test-framework,common.compile-test"/>
-
- <target name="test" depends="compile-test,junit-mkdir,junit-sequential,junit-parallel"/>
+ <target name="compile-core" depends="compile-solrj,common-solr.compile-core"/>
<target name="dist-maven" depends="jar-core,javadocs,jar-src,contrib-build.dist-maven">
<m2-deploy-with-pom-template pom.xml="${common-solr.dir}/lib/solr-commons-csv-pom.xml.template"
@@ -36,4 +29,4 @@
<m2-deploy-with-pom-template pom.xml="${common-solr.dir}/lib/apache-solr-noggit-pom.xml.template"
jar.file="${common-solr.dir}/lib/apache-solr-noggit-r1099557.jar" />
</target>
-</project>
\ No newline at end of file
+</project>
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java Tue Aug 23 14:06:58 2011
@@ -37,7 +37,7 @@ import org.apache.lucene.analysis.synony
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.analysis.synonym.SolrSynonymParser;
import org.apache.lucene.analysis.synonym.WordnetSynonymParser;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.Version;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.common.SolrException;
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java Tue Aug 23 14:06:58 2011
@@ -51,7 +51,7 @@ public class UAX29URLEmailTokenizerFacto
}
public UAX29URLEmailTokenizer create(Reader input) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(input);
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, input);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/CoreContainer.java Tue Aug 23 14:06:58 2011
@@ -716,7 +716,7 @@ public class CoreContainer
File instanceDir = new File(cd.getInstanceDir());
if (!instanceDir.isAbsolute()) {
- instanceDir = new File(getSolrHome(), instanceDir.getName());
+ instanceDir = new File(getSolrHome(), cd.getInstanceDir());
}
SolrResourceLoader solrLoader;
@@ -903,8 +903,8 @@ public class CoreContainer
List<SolrCoreXMLDef> solrCoreXMLDefs = new ArrayList<SolrCoreXMLDef>();
synchronized (cores) {
- Map<String,String> coreAttribs = new HashMap<String,String>();
for (SolrCore solrCore : cores.values()) {
+ Map<String,String> coreAttribs = new HashMap<String,String>();
CoreDescriptor dcore = solrCore.getCoreDescriptor();
coreAttribs.put("name", dcore.name.equals("") ? defaultCoreName
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Tue Aug 23 14:06:58 2011
@@ -1,4 +1,5 @@
package org.apache.solr.core;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,33 +17,71 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import java.io.File;
+import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.store.Directory;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
/**
- * Provides access to a Directory implementation.
- *
+ * Provides access to a Directory implementation. You must release every
+ * Directory that you get.
*/
-public abstract class DirectoryFactory implements NamedListInitializedPlugin {
-
+public abstract class DirectoryFactory implements NamedListInitializedPlugin,
+ Closeable {
+
/**
- * Opens a Lucene directory
+ * Close the this and all of the Directories it contains.
*
* @throws IOException
*/
- public abstract Directory open(String path) throws IOException;
+ public abstract void close() throws IOException;
- public boolean exists(String path) {
- // back compat behavior
- File dirFile = new File(path);
- return dirFile.canRead();
- }
-
+ /**
+ * Creates a new Directory for a given path.
+ *
+ * @throws IOException
+ */
+ protected abstract Directory create(String path) throws IOException;
+
+ /**
+ * Returns true if a Directory exists for a given path.
+ *
+ */
+ public abstract boolean exists(String path);
+
+ /**
+ * Returns the Directory for a given path, using the specified rawLockType.
+ * Will return the same Directory instance for the same path.
+ *
+ * @throws IOException
+ */
+ public abstract Directory get(String path, String rawLockType)
+ throws IOException;
+
+ /**
+ * Returns the Directory for a given path, using the specified rawLockType.
+ * Will return the same Directory instance for the same path unless forceNew,
+ * in which case a new Directory is returned.
+ *
+ * @throws IOException
+ */
+ public abstract Directory get(String path, String rawLockType,
+ boolean forceNew) throws IOException;
+
+ /**
+ * Increment the number of references to the given Directory. You must call
+ * release for every call to this method.
+ *
+ */
+ public abstract void incRef(Directory directory);
+
+ /**
+ * Releases the Directory so that it may be closed when it is no longer
+ * referenced.
+ *
+ * @throws IOException
+ */
+ public abstract void release(Directory directory) throws IOException;
- public void init(NamedList args) {
- }
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java Tue Aug 23 14:06:58 2011
@@ -38,13 +38,23 @@ import java.io.IOException;
* </ul>
*
**/
-public class MMapDirectoryFactory extends DirectoryFactory {
+public class MMapDirectoryFactory extends CachingDirectoryFactory {
private transient static Logger log = LoggerFactory.getLogger(MMapDirectoryFactory.class);
boolean unmapHack;
private int maxChunk;
@Override
- public Directory open(String path) throws IOException {
+ public void init(NamedList args) {
+ SolrParams params = SolrParams.toSolrParams( args );
+ maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_BUFF);
+ if (maxChunk <= 0){
+ throw new IllegalArgumentException("maxChunk must be greater than 0");
+ }
+ unmapHack = params.getBool("unmap", true);
+ }
+
+ @Override
+ protected Directory create(String path) throws IOException {
MMapDirectory mapDirectory = new MMapDirectory(new File(path));
try {
mapDirectory.setUseUnmap(unmapHack);
@@ -54,14 +64,4 @@ public class MMapDirectoryFactory extend
mapDirectory.setMaxChunkSize(maxChunk);
return mapDirectory;
}
-
- @Override
- public void init(NamedList args) {
- SolrParams params = SolrParams.toSolrParams( args );
- maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_BUFF);
- if (maxChunk <= 0){
- throw new IllegalArgumentException("maxChunk must be greater than 0");
- }
- unmapHack = params.getBool("unmap", true);
- }
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java Tue Aug 23 14:06:58 2011
@@ -19,37 +19,30 @@ package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
/**
- * Directory provider for using lucene RAMDirectory
+ * Factory to instantiate {@link org.apache.lucene.store.RAMDirectory}
*/
public class RAMDirectoryFactory extends StandardDirectoryFactory {
- private static Map<String, RefCntRamDirectory> directories = new HashMap<String, RefCntRamDirectory>();
@Override
- public Directory open(String path) throws IOException {
- synchronized (RAMDirectoryFactory.class) {
- RefCntRamDirectory directory = directories.get(path);
- if (directory == null || !directory.isOpen()) {
- directory = (RefCntRamDirectory) openNew(path);
- directories.put(path, directory);
- } else {
- directory.incRef();
- }
-
- return directory;
- }
+ protected Directory create(String path) throws IOException {
+ return new RAMDirectory();
}
@Override
public boolean exists(String path) {
- synchronized (RAMDirectoryFactory.class) {
- RefCntRamDirectory directory = directories.get(path);
- if (directory == null || !directory.isOpen()) {
+ String fullPath = new File(path).getAbsolutePath();
+ synchronized (DirectoryFactory.class) {
+ CacheValue cacheValue = byPathCache.get(fullPath);
+ Directory directory = null;
+ if (cacheValue != null) {
+ directory = cacheValue.directory;
+ }
+ if (directory == null) {
return false;
} else {
return true;
@@ -57,19 +50,4 @@ public class RAMDirectoryFactory extends
}
}
- /**
- * Non-public for unit-test access only. Do not use directly
- */
- Directory openNew(String path) throws IOException {
- Directory directory;
- File dirFile = new File(path);
- boolean indexExists = dirFile.canRead();
- if (indexExists) {
- Directory dir = super.open(path);
- directory = new RefCntRamDirectory(dir);
- } else {
- directory = new RefCntRamDirectory();
- }
- return directory;
- }
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java Tue Aug 23 14:06:58 2011
@@ -27,10 +27,10 @@ import java.io.IOException;
* Factory to instantiate {@link org.apache.lucene.store.SimpleFSDirectory}
*
**/
-public class SimpleFSDirectoryFactory extends DirectoryFactory {
+public class SimpleFSDirectoryFactory extends CachingDirectoryFactory {
@Override
- public Directory open(String path) throws IOException {
+ protected Directory create(String path) throws IOException {
return new SimpleFSDirectory(new File(path));
}
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/SolrCore.java Tue Aug 23 14:06:58 2011
@@ -320,7 +320,7 @@ public final class SolrCore implements S
// gets a non-caching searcher
public SolrIndexSearcher newSearcher(String name, boolean readOnly) throws IOException {
- return new SolrIndexSearcher(this, schema, name, directoryFactory.open(getIndexDir()), readOnly, false);
+ return new SolrIndexSearcher(this, getNewIndexDir(), schema, getSolrConfig().mainIndexConfig, name, readOnly, false, directoryFactory);
}
@@ -355,7 +355,6 @@ public final class SolrCore implements S
void initIndex() {
try {
- initDirectoryFactory();
String indexDir = getNewIndexDir();
boolean indexExists = getDirectoryFactory().exists(indexDir);
boolean firstTime;
@@ -369,13 +368,13 @@ public final class SolrCore implements S
if (indexExists && firstTime && removeLocks) {
// to remove locks, the directory must already exist... so we create it
// if it didn't exist already...
- Directory dir = SolrIndexWriter.getDirectory(indexDir, getDirectoryFactory(), solrConfig.mainIndexConfig);
+ Directory dir = directoryFactory.get(indexDir, getSolrConfig().mainIndexConfig.lockType);
if (dir != null) {
if (IndexWriter.isLocked(dir)) {
log.warn(logid+"WARNING: Solr index directory '" + indexDir+ "' is locked. Unlocking...");
IndexWriter.unlock(dir);
}
- dir.close();
+ directoryFactory.release(dir);
}
}
@@ -384,7 +383,7 @@ public final class SolrCore implements S
log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist."
+ " Creating new index...");
- SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codecProvider);
+ SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codecProvider, false);
writer.close();
}
@@ -490,19 +489,18 @@ public final class SolrCore implements S
* @since solr 1.0
*/
public SolrCore(String dataDir, IndexSchema schema) throws ParserConfigurationException, IOException, SAXException {
- this(null, dataDir, new SolrConfig(), schema, null );
+ this(null, dataDir, new SolrConfig(), schema, null);
}
/**
* Creates a new core and register it in the list of cores.
* If a core with the same name already exists, it will be stopped and replaced by this one.
- *
+ *
* @param name
* @param dataDir the index directory
* @param config a solr config instance
* @param schema a solr schema instance
- * @param cd
- *
+ *
* @since solr 1.3
*/
public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd) {
@@ -559,6 +557,13 @@ public final class SolrCore implements S
initDeletionPolicy();
this.codecProvider = initCodecProvider(solrConfig, schema);
+
+ if (updateHandler == null) {
+ initDirectoryFactory();
+ } else {
+ directoryFactory = updateHandler.getIndexWriterProvider().getDirectoryFactory();
+ }
+
initIndex();
initWriters();
@@ -601,7 +606,6 @@ public final class SolrCore implements S
this.updateHandler = createUpdateHandler(updateHandlerClass == null ? DirectUpdateHandler2.class
.getName() : updateHandlerClass);
} else {
-
this.updateHandler = createUpdateHandler(
updateHandlerClass == null ? DirectUpdateHandler2.class.getName()
: updateHandlerClass, updateHandler);
@@ -737,11 +741,7 @@ public final class SolrCore implements S
} catch (Exception e) {
SolrException.log(log, e);
}
- try {
- updateHandler.close();
- } catch (Exception e) {
- SolrException.log(log,e);
- }
+
try {
searcherExecutor.shutdown();
if (!searcherExecutor.awaitTermination(60, TimeUnit.SECONDS)) {
@@ -763,6 +763,12 @@ public final class SolrCore implements S
SolrException.log(log,e);
}
+ try {
+ updateHandler.close();
+ } catch (Exception e) {
+ SolrException.log(log,e);
+ }
+
if( closeHooks != null ) {
for( CloseHook hook : closeHooks ) {
try {
@@ -1114,12 +1120,11 @@ public final class SolrCore implements S
currentReader.incRef();
}
- tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
+ tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true, true, directoryFactory);
}
} else {
- IndexReader reader = getIndexReaderFactory().newReader(getDirectoryFactory().open(newIndexDir), true);
- tmp = new SolrIndexSearcher(this, schema, "main", reader, true, true);
+ tmp = new SolrIndexSearcher(this, newIndexDir, schema, getSolrConfig().mainIndexConfig, "main", true, true, directoryFactory);
}
} catch (Throwable th) {
synchronized(searcherLock) {
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java Tue Aug 23 14:06:58 2011
@@ -23,13 +23,14 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory;
/**
- * Directory provider which mimics original Solr FSDirectory based behavior.
+ * Directory provider which mimics original Solr
+ * {@link org.apache.lucene.store.FSDirectory} based behavior.
*
*/
-public class StandardDirectoryFactory extends DirectoryFactory {
+public class StandardDirectoryFactory extends CachingDirectoryFactory {
@Override
- public Directory open(String path) throws IOException {
+ protected Directory create(String path) throws IOException {
return FSDirectory.open(new File(path));
}
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue Aug 23 14:06:58 2011
@@ -34,13 +34,18 @@ import org.apache.commons.io.IOUtils;
import java.util.regex.Pattern;
import java.util.List;
+import java.util.HashMap;
+import java.util.Iterator;
import java.io.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class CSVRequestHandler extends ContentStreamHandlerBase {
+ public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
@Override
protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
@@ -83,16 +88,20 @@ abstract class CSVLoader extends Content
public static final String ENCAPSULATOR="encapsulator";
public static final String ESCAPE="escape";
public static final String OVERWRITE="overwrite";
+ public static final String LITERALS_PREFIX = "literal.";
private static Pattern colonSplit = Pattern.compile(":");
private static Pattern commaSplit = Pattern.compile(",");
+
+ public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
final IndexSchema schema;
final SolrParams params;
final CSVStrategy strategy;
final UpdateRequestProcessor processor;
-
+ // hashmap to save any literal fields and their values
+ HashMap <SchemaField, String> literals;
String[] fieldnames;
SchemaField[] fields;
CSVLoader.FieldAdder[] adders;
@@ -189,6 +198,7 @@ abstract class CSVLoader extends Content
this.processor = processor;
this.params = req.getParams();
schema = req.getSchema();
+ this.literals = new HashMap<SchemaField, String>();
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite=params.getBool(OVERWRITE,true);
@@ -289,7 +299,7 @@ abstract class CSVLoader extends Content
adders[i] = new CSVLoader.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
}
}
-
+
if (params.getFieldBool(fname,TRIM,false)) {
adders[i] = new CSVLoader.FieldTrimmer(adders[i]);
}
@@ -306,6 +316,18 @@ abstract class CSVLoader extends Content
adders[i] = new CSVLoader.FieldSplitter(fstrat, adders[i]);
}
}
+ // look for any literal fields - literal.foo=xyzzy
+ Iterator<String> paramNames = params.getParameterNamesIterator();
+ while (paramNames.hasNext()) {
+ String pname = paramNames.next();
+ if (!pname.startsWith(LITERALS_PREFIX)) continue;
+
+ String name = pname.substring(LITERALS_PREFIX.length());
+ SchemaField sf = schema.getFieldOrNull(name);
+ if(sf == null)
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
+ literals.put(sf, params.get(pname));
+ }
}
private void input_err(String msg, String[] line, int lineno) {
@@ -397,6 +419,13 @@ abstract class CSVLoader extends Content
adders[i].add(doc, line, i, val);
}
+ // add any literals
+ for (SchemaField sf : literals.keySet()) {
+ String fn = sf.getName();
+ String val = literals.get(sf);
+ doc.addField(fn, val);
+ }
+
template.solrDoc = doc;
processor.processAdd(template);
}
@@ -411,7 +440,7 @@ class SingleThreadedCSVLoader extends CS
@Override
void addDoc(int line, String[] vals) throws IOException {
- templateAdd.indexedId = null;
+ templateAdd.clear();
SolrInputDocument doc = new SolrInputDocument();
doAdd(line, vals, doc, templateAdd);
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java Tue Aug 23 14:06:58 2011
@@ -367,7 +367,8 @@ public class MoreLikeThisHandler extends
public DocListAndSet getMoreLikeThis( Reader reader, int start, int rows, List<Query> filters, List<InterestingTerm> terms, int flags ) throws IOException
{
- rawMLTQuery = mlt.like(reader);
+ // analyzing with the first field: previous (stupid) behavior
+ rawMLTQuery = mlt.like(reader, mlt.getFieldNames()[0]);
boostedMLTQuery = getBoostedQuery( rawMLTQuery );
if( terms != null ) {
fillInterestingTermsFromMLTQuery( boostedMLTQuery, terms );
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Tue Aug 23 14:06:58 2011
@@ -16,6 +16,30 @@
*/
package org.apache.solr.handler;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+import java.util.zip.DeflaterOutputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexReader;
@@ -26,28 +50,21 @@ import org.apache.solr.common.util.FastO
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.core.*;
+import org.apache.solr.core.CloseHook;
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrDeletionPolicy;
+import org.apache.solr.core.SolrEventListener;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.BinaryQueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.NumberUtils;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.plugin.SolrCoreAware;
-import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.text.NumberFormat;
-import java.util.*;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.zip.Adler32;
-import java.util.zip.Checksum;
-import java.util.zip.DeflaterOutputStream;
-
/**
* <p> A Handler which provides a REST API for replication and serves replication requests from Slaves. <p/> </p>
* <p>When running on the master, it provides the following commands <ol> <li>Get the current replicatable index version
@@ -423,18 +440,7 @@ public class ReplicationHandler extends
}
long getIndexSize() {
- return computeIndexSize(new File(core.getIndexDir()));
- }
-
- private long computeIndexSize(File f) {
- if (f.isFile())
- return f.length();
- File[] files = f.listFiles();
- long size = 0;
- if (files != null && files.length > 0) {
- for (File file : files) size += file.length();
- }
- return size;
+ return FileUtils.sizeOfDirectory(new File(core.getIndexDir()));
}
/**
@@ -468,20 +474,6 @@ public class ReplicationHandler extends
return "$Revision$";
}
- String readableSize(long size) {
- NumberFormat formatter = NumberFormat.getNumberInstance();
- formatter.setMaximumFractionDigits(2);
- if (size / (1024 * 1024 * 1024) > 0) {
- return formatter.format(size * 1.0d / (1024 * 1024 * 1024)) + " GB";
- } else if (size / (1024 * 1024) > 0) {
- return formatter.format(size * 1.0d / (1024 * 1024)) + " MB";
- } else if (size / 1024 > 0) {
- return formatter.format(size * 1.0d / 1024) + " KB";
- } else {
- return String.valueOf(size) + " bytes";
- }
- }
-
private long[] getIndexVersion() {
long version[] = new long[2];
RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
@@ -501,7 +493,7 @@ public class ReplicationHandler extends
public NamedList getStatistics() {
NamedList list = super.getStatistics();
if (core != null) {
- list.add("indexSize", readableSize(getIndexSize()));
+ list.add("indexSize", NumberUtils.readableSize(getIndexSize()));
long[] versionGen = getIndexVersion();
list.add("indexVersion", versionGen[0]);
list.add(GENERATION, versionGen[1]);
@@ -555,7 +547,7 @@ public class ReplicationHandler extends
NamedList<Object> master = new SimpleOrderedMap<Object>();
NamedList<Object> slave = new SimpleOrderedMap<Object>();
- details.add("indexSize", readableSize(getIndexSize()));
+ details.add("indexSize", NumberUtils.readableSize(getIndexSize()));
details.add("indexPath", core.getIndexDir());
details.add(CMD_SHOW_COMMITS, getCommits());
details.add("isMaster", String.valueOf(isMaster));
@@ -631,7 +623,7 @@ public class ReplicationHandler extends
slave.add("filesToDownload", filesToDownload);
slave.add("numFilesToDownload", String.valueOf(filesToDownload.size()));
- slave.add("bytesToDownload", readableSize(bytesToDownload));
+ slave.add("bytesToDownload", NumberUtils.readableSize(bytesToDownload));
long bytesDownloaded = 0;
List<String> filesDownloaded = new ArrayList<String>();
@@ -681,13 +673,13 @@ public class ReplicationHandler extends
downloadSpeed = (bytesDownloaded / elapsed);
if (currFile != null)
slave.add("currentFile", currFile);
- slave.add("currentFileSize", readableSize(currFileSize));
- slave.add("currentFileSizeDownloaded", readableSize(currFileSizeDownloaded));
+ slave.add("currentFileSize", NumberUtils.readableSize(currFileSize));
+ slave.add("currentFileSizeDownloaded", NumberUtils.readableSize(currFileSizeDownloaded));
slave.add("currentFileSizePercent", String.valueOf(percentDownloaded));
- slave.add("bytesDownloaded", readableSize(bytesDownloaded));
+ slave.add("bytesDownloaded", NumberUtils.readableSize(bytesDownloaded));
slave.add("totalPercent", String.valueOf(totalPercent));
slave.add("timeRemaining", String.valueOf(estimatedTimeRemaining) + "s");
- slave.add("downloadSpeed", readableSize(downloadSpeed));
+ slave.add("downloadSpeed", NumberUtils.readableSize(downloadSpeed));
} catch (Exception e) {
LOG.error("Exception while writing replication details: ", e);
}
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/XMLLoader.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/XMLLoader.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/XMLLoader.java Tue Aug 23 14:06:58 2011
@@ -211,8 +211,7 @@ class XMLLoader extends ContentStreamLoa
"unexpected XML tag /delete/" + currTag);
}
processor.processDelete(deleteCmd);
- deleteCmd.id = null;
- deleteCmd.query = null;
+ deleteCmd.clear();
break;
// Add everything to the text
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Tue Aug 23 14:06:58 2011
@@ -17,7 +17,9 @@
package org.apache.solr.handler.admin;
+import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.SolrException;
@@ -33,6 +35,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.NumberUtils;
import org.apache.solr.util.RefCounted;
import org.apache.solr.update.MergeIndexesCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
@@ -43,6 +46,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Date;
+import java.util.Iterator;
+import java.util.Properties;
/**
*
@@ -181,6 +186,7 @@ public class CoreAdminHandler extends Re
RefCounted<SolrIndexSearcher>[] searchers = null;
// stores readers created from indexDir param values
IndexReader[] readersToBeClosed = null;
+ Directory[] dirsToBeReleased = null;
if (core != null) {
try {
String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR);
@@ -201,9 +207,12 @@ public class CoreAdminHandler extends Re
}
} else {
readersToBeClosed = new IndexReader[dirNames.length];
+ dirsToBeReleased = new Directory[dirNames.length];
DirectoryFactory dirFactory = core.getDirectoryFactory();
for (int i = 0; i < dirNames.length; i++) {
- readersToBeClosed[i] = IndexReader.open(dirFactory.open(dirNames[i]), true);
+ Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().mainIndexConfig.lockType);
+ dirsToBeReleased[i] = dir;
+ readersToBeClosed[i] = IndexReader.open(dir, true);
}
}
@@ -239,6 +248,12 @@ public class CoreAdminHandler extends Re
}
}
if (readersToBeClosed != null) IOUtils.closeSafely(true, readersToBeClosed);
+ if (dirsToBeReleased != null) {
+ for (Directory dir : dirsToBeReleased) {
+ DirectoryFactory dirFactory = core.getDirectoryFactory();
+ dirFactory.release(dir);
+ }
+ }
if (wrappedReq != null) wrappedReq.close();
core.close();
}
@@ -306,8 +321,20 @@ public class CoreAdminHandler extends Re
if (opts != null)
cd.setShardId(opts);
}
-
- dcore.setCoreProperties(null);
+
+ // Process all property.name=value parameters and set them as name=value core properties
+ Properties coreProperties = new Properties();
+ Iterator<String> parameterNamesIterator = params.getParameterNamesIterator();
+ while (parameterNamesIterator.hasNext()) {
+ String parameterName = parameterNamesIterator.next();
+ if(parameterName.startsWith(CoreAdminParams.PROPERTY_PREFIX)) {
+ String parameterValue = params.get(parameterName);
+ String propertyName = parameterName.substring(CoreAdminParams.PROPERTY_PREFIX.length()); // skip prefix
+ coreProperties.put(propertyName, parameterValue);
+ }
+ }
+ dcore.setCoreProperties(coreProperties);
+
SolrCore core = coreContainer.create(dcore);
coreContainer.register(name, core, false);
rsp.add("core", core.getName());
@@ -401,12 +428,20 @@ public class CoreAdminHandler extends Re
@Override
public void postClose(SolrCore core) {
File dataDir = new File(core.getIndexDir());
- for (File file : dataDir.listFiles()) {
- if (!file.delete()) {
- log.error(file.getAbsolutePath() + " could not be deleted on core unload");
+ File[] files = dataDir.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (!file.delete()) {
+ log.error(file.getAbsolutePath()
+ + " could not be deleted on core unload");
+ }
}
+ if (!dataDir.delete()) log.error(dataDir.getAbsolutePath()
+ + " could not be deleted on core unload");
+ } else {
+ log.error(dataDir.getAbsolutePath()
+ + " could not be deleted on core unload");
}
- if (!dataDir.delete()) log.error(dataDir.getAbsolutePath() + " could not be deleted on core unload");
}
});
}
@@ -530,7 +565,11 @@ public class CoreAdminHandler extends Re
info.add("uptime", System.currentTimeMillis() - core.getStartTime());
RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
try {
- info.add("index", LukeRequestHandler.getIndexInfo(searcher.get().getIndexReader(), false));
+ SimpleOrderedMap<Object> indexInfo = LukeRequestHandler.getIndexInfo(searcher.get().getIndexReader(), false);
+ long size = getIndexSize(core);
+ indexInfo.add("sizeInBytes", size);
+ indexInfo.add("size", NumberUtils.readableSize(size));
+ info.add("index", indexInfo);
} finally {
searcher.decref();
}
@@ -540,6 +579,10 @@ public class CoreAdminHandler extends Re
}
return info;
}
+
+ private long getIndexSize(SolrCore core) {
+ return FileUtils.sizeOfDirectory(new File(core.getIndexDir()));
+ }
protected static String normalizePath(String path) {
if (path == null)
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Tue Aug 23 14:06:58 2011
@@ -204,9 +204,8 @@ public class LukeRequestHandler extends
flags.append( (f != null && f.storeTermPositions() ) ? FieldFlag.TERM_VECTOR_POSITION.getAbbreviation() : '-' );
flags.append( (f != null && f.omitNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' );
flags.append( (f != null &&
- f.indexOptions() == IndexOptions.DOCS_ONLY) ? FieldFlag.OMIT_TF.getAbbreviation() : '-' );
- flags.append( (f != null &&
- f.indexOptions() == IndexOptions.DOCS_AND_FREQS) ? FieldFlag.OMIT_POSITIONS.getAbbreviation() : '-' );
+ f.omitTermFreqAndPositions() ) ? FieldFlag.OMIT_TF.getAbbreviation() : '-' );
+ flags.append( (f != null && f.omitPositions() ) ? FieldFlag.OMIT_POSITIONS.getAbbreviation() : '-' );
flags.append( (lazy) ? FieldFlag.LAZY.getAbbreviation() : '-' );
flags.append( (binary) ? FieldFlag.BINARY.getAbbreviation() : '-' );
flags.append( (f != null && f.sortMissingFirst() ) ? FieldFlag.SORT_MISSING_FIRST.getAbbreviation() : '-' );
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java Tue Aug 23 14:06:58 2011
@@ -289,10 +289,6 @@ public class FacetComponent extends Sear
int shardNum = rb.getShardNum(srsp.getShard());
NamedList facet_counts = (NamedList)srsp.getSolrResponse().getResponse().get("facet_counts");
- @SuppressWarnings("unchecked")
- List<String> excepts = (List<String>)facet_counts.get("exception");
- fi.addExceptions(excepts);
-
// handle facet queries
NamedList facet_queries = (NamedList)facet_counts.get("facet_queries");
if (facet_queries != null) {
@@ -476,10 +472,6 @@ public class FacetComponent extends Sear
// int shardNum = rb.getShardNum(srsp.shard);
NamedList facet_counts = (NamedList)srsp.getSolrResponse().getResponse().get("facet_counts");
NamedList facet_fields = (NamedList)facet_counts.get("facet_fields");
-
- @SuppressWarnings("unchecked")
- List<String> excepts = (List<String>)facet_counts.get("exception");
- fi.addExceptions(excepts);
if (facet_fields == null) continue; // this can happen when there's an exception
@@ -511,10 +503,6 @@ public class FacetComponent extends Sear
NamedList<Object> facet_counts = new SimpleOrderedMap<Object>();
- if (fi.exceptionList != null) {
- facet_counts.add("exception",fi.exceptionList);
- }
-
NamedList<Number> facet_queries = new SimpleOrderedMap<Number>();
facet_counts.add("facet_queries",facet_queries);
for (QueryFacet qf : fi.queryFacets.values()) {
@@ -635,8 +623,6 @@ public class FacetComponent extends Sear
public SimpleOrderedMap<SimpleOrderedMap<Object>> rangeFacets
= new SimpleOrderedMap<SimpleOrderedMap<Object>>();
- public List<String> exceptionList;
-
void parse(SolrParams params, ResponseBuilder rb) {
queryFacets = new LinkedHashMap<String,QueryFacet>();
facets = new LinkedHashMap<String,DistribFieldFacet>();
@@ -658,12 +644,6 @@ public class FacetComponent extends Sear
}
}
}
-
- public void addExceptions(List<String> exceptions) {
- if (exceptions == null) return;
- if (exceptionList == null) exceptionList = new ArrayList<String>();
- exceptionList.addAll(exceptions);
- }
}
/**
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Tue Aug 23 14:06:58 2011
@@ -320,6 +320,7 @@ public class QueryComponent extends Sear
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
String groupSortStr = params.get(GroupParams.GROUP_SORT);
boolean main = params.getBool(GroupParams.GROUP_MAIN, false);
+ boolean truncateGroups = params.getBool(GroupParams.GROUP_TRUNCATE, false);
String formatStr = params.get(GroupParams.GROUP_FORMAT, Grouping.Format.grouped.name());
Grouping.Format defaultFormat;
@@ -346,7 +347,8 @@ public class QueryComponent extends Sear
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
.setDocsPerGroupDefault(docsPerGroupDefault)
- .setGroupOffsetDefault(groupOffsetDefault);
+ .setGroupOffsetDefault(groupOffsetDefault)
+ .setGetGroupedDocSet(truncateGroups);
if (fields != null) {
for (String field : fields) {
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java Tue Aug 23 14:06:58 2011
@@ -473,6 +473,7 @@ public class SpellCheckComponent extends
private Collection<Token> getTokens(String q, Analyzer analyzer) throws IOException {
Collection<Token> result = new ArrayList<Token>();
+ assert analyzer != null;
TokenStream ts = analyzer.reusableTokenStream("", new StringReader(q));
ts.reset();
// TODO: support custom attributes
@@ -589,6 +590,8 @@ public class SpellCheckComponent extends
if (initParams.getName(i).equals("spellchecker")) {
NamedList spellchecker = (NamedList) initParams.getVal(i);
String className = (String) spellchecker.get("classname");
+ // TODO: this is a little bit sneaky: warn if class isnt supplied
+ // so that its mandatory in a future release?
if (className == null)
className = IndexBasedSpellChecker.class.getName();
SolrResourceLoader loader = core.getResourceLoader();
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Tue Aug 23 14:06:58 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.util.packed.Dir
import org.apache.lucene.util.packed.Direct8;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.RequiredSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -191,26 +192,16 @@ public class SimpleFacets {
facetResponse.add("facet_dates", getFacetDateCounts());
facetResponse.add("facet_ranges", getFacetRangeCounts());
- } catch (Exception e) {
+ } catch (IOException e) {
SolrException.logOnce(SolrCore.log, "Exception during facet counts", e);
- addException("Exception during facet counts", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (ParseException e) {
+ SolrException.logOnce(SolrCore.log, "Exception during facet counts", e);
+ throw new SolrException(ErrorCode.BAD_REQUEST, e);
}
return facetResponse;
}
- public void addException(String msg, Exception e) {
- @SuppressWarnings("unchecked")
- List<String> exceptions = (List<String>)facetResponse.get("exception");
-
- if (exceptions == null) {
- exceptions = new ArrayList<String>();
- facetResponse.add("exception", exceptions);
- }
-
- String entry = msg + '\n' + SolrException.toStr(e);
- exceptions.add(entry);
- }
-
/**
* Returns a list of facet counts for each of the facet queries
* specified in the params
@@ -232,18 +223,11 @@ public class SimpleFacets {
if (null != facetQs && 0 != facetQs.length) {
for (String q : facetQs) {
- try {
- parseParams(FacetParams.FACET_QUERY, q);
+ parseParams(FacetParams.FACET_QUERY, q);
- // TODO: slight optimization would prevent double-parsing of any localParams
- Query qobj = QParser.getParser(q, null, req).getQuery();
- res.add(key, searcher.numDocs(qobj, base));
- }
- catch (Exception e) {
- String msg = "Exception during facet.query of " + q;
- SolrException.logOnce(SolrCore.log, msg, e);
- addException(msg , e);
- }
+ // TODO: slight optimization would prevent double-parsing of any localParams
+ Query qobj = QParser.getParser(q, null, req).getQuery();
+ res.add(key, searcher.numDocs(qobj, base));
}
}
@@ -347,18 +331,12 @@ public class SimpleFacets {
String[] facetFs = params.getParams(FacetParams.FACET_FIELD);
if (null != facetFs) {
for (String f : facetFs) {
- try {
- parseParams(FacetParams.FACET_FIELD, f);
- String termList = localParams == null ? null : localParams.get(CommonParams.TERMS);
- if (termList != null) {
- res.add(key, getListedTermCounts(facetValue, termList));
- } else {
- res.add(key, getTermCounts(facetValue));
- }
- } catch (Exception e) {
- String msg = "Exception during facet.field of " + f;
- SolrException.logOnce(SolrCore.log, msg, e);
- addException(msg , e);
+ parseParams(FacetParams.FACET_FIELD, f);
+ String termList = localParams == null ? null : localParams.get(CommonParams.TERMS);
+ if (termList != null) {
+ res.add(key, getListedTermCounts(facetValue, termList));
+ } else {
+ res.add(key, getTermCounts(facetValue));
}
}
}
@@ -780,13 +758,7 @@ public class SimpleFacets {
if (null == fields || 0 == fields.length) return resOuter;
for (String f : fields) {
- try {
- getFacetDateCounts(f, resOuter);
- } catch (Exception e) {
- String msg = "Exception during facet.date of " + f;
- SolrException.logOnce(SolrCore.log, msg, e);
- addException(msg , e);
- }
+ getFacetDateCounts(f, resOuter);
}
return resOuter;
@@ -955,20 +927,14 @@ public class SimpleFacets {
* @see FacetParams#FACET_RANGE
*/
- public NamedList<Object> getFacetRangeCounts() {
+ public NamedList<Object> getFacetRangeCounts() throws IOException, ParseException {
final NamedList<Object> resOuter = new SimpleOrderedMap<Object>();
final String[] fields = params.getParams(FacetParams.FACET_RANGE);
if (null == fields || 0 == fields.length) return resOuter;
for (String f : fields) {
- try {
- getFacetRangeCounts(f, resOuter);
- } catch (Exception e) {
- String msg = "Exception during facet.range of " + f;
- SolrException.logOnce(SolrCore.log, msg, e);
- addException(msg , e);
- }
+ getFacetRangeCounts(f, resOuter);
}
return resOuter;
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java Tue Aug 23 14:06:58 2011
@@ -89,12 +89,15 @@ public class BinaryResponseWriter implem
}
if (o instanceof SolrDocument) {
- // Remove any fields that were not requested
- // This typically happens when distributed search adds extra fields to an internal request
+ // Remove any fields that were not requested.
+ // This typically happens when distributed search adds
+ // extra fields to an internal request
SolrDocument doc = (SolrDocument)o;
- for( String fname : doc.getFieldNames() ) {
- if( !returnFields.wantsField( fname ) ) {
- doc.removeFields( fname );
+ Iterator<Map.Entry<String, Object>> i = doc.iterator();
+ while ( i.hasNext() ) {
+ String fname = i.next().getKey();
+ if ( !returnFields.wantsField( fname ) ) {
+ i.remove();
}
}
return doc;
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java Tue Aug 23 14:06:58 2011
@@ -203,10 +203,6 @@ public abstract class TextResponseWriter
{
SolrDocument out = new SolrDocument();
for( Fieldable f : doc.getFields() ) {
- if( "gack_i".equals( f.name() ) ) {
- System.out.println( f );
- }
-
// Make sure multivalued fields are represented as lists
Object existing = out.get(f.name());
if (existing == null) {
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldType.java Tue Aug 23 14:06:58 2011
@@ -250,9 +250,10 @@ public abstract class FieldType extends
}
if (val==null) return null;
- return createField(field.getName(), val, getFieldStore(field, val),
- getFieldIndex(field, val), getFieldTermVec(field, val), field.omitNorms(),
- field.indexOptions(), boost);
+ return createField(field.getName(), val,
+ getFieldStore(field, val), getFieldIndex(field, val),
+ getFieldTermVec(field, val), field.omitNorms(),
+ getIndexOptions(field, val), boost);
}
@@ -276,9 +277,11 @@ public abstract class FieldType extends
storage,
index,
vec);
- f.setOmitNorms(omitNorms);
- f.setIndexOptions(options);
- f.setBoost(boost);
+ if (index.isIndexed()) {
+ f.setOmitNorms(omitNorms);
+ f.setIndexOptions(options);
+ f.setBoost(boost);
+ }
return f;
}
@@ -320,6 +323,16 @@ public abstract class FieldType extends
return field.indexed() ? (isTokenized() ? Field.Index.ANALYZED :
Field.Index.NOT_ANALYZED) : Field.Index.NO;
}
+ protected IndexOptions getIndexOptions(SchemaField field,
+ String internalVal) {
+ IndexOptions options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ if (field.omitTermFreqAndPositions()) {
+ options = IndexOptions.DOCS_ONLY;
+ } else if (field.omitPositions()) {
+ options = IndexOptions.DOCS_AND_FREQS;
+ }
+ return options;
+ }
/**
* Convert an external value (from XML update command or from query string)
Copied: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java (from r1160237, lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java?p2=lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java&p1=lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java&r1=1160237&r2=1160700&rev=1160700&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java Tue Aug 23 14:06:58 2011
@@ -18,7 +18,7 @@
package org.apache.solr.schema;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Version;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.common.SolrException;