You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2011/03/15 22:35:35 UTC
svn commit: r1081952 [12/17] - in /lucene/dev/branches/bulkpostings: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/
dev-tools/idea/lucene/ dev-tools/idea/lucene/contrib/ant/
dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/luc...
Modified: lucene/dev/branches/bulkpostings/solr/contrib/clustering/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/clustering/build.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/clustering/build.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/clustering/build.xml Tue Mar 15 21:35:17 2011
@@ -30,7 +30,7 @@
<property name="example.local" value="example"/>
<path id="common.classpath">
- <fileset dir="lib"/>
+ <fileset dir="lib" includes="*.jar"/>
<pathelement location="${solr-path}/build/solr"/>
<pathelement location="${solr-path}/build/solrj"/>
<path refid="lucene.classpath"/>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/clustering/src/test/resources/solr-clustering/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/clustering/src/test/resources/solr-clustering/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/clustering/src/test/resources/solr-clustering/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/clustering/src/test/resources/solr-clustering/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -18,19 +18,11 @@
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
- <!-- Set this to 'false' if you want solr to continue working after it has
- encountered an severe configuration error. In a production environment,
- you may want solr to keep working even if one handler is mis-configured.
-
- You may also set this to false using by setting the system property:
- -Dsolr.abortOnConfigurationError=false
- -->
- <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<indexDefaults>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/CHANGES.txt?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/CHANGES.txt Tue Mar 15 21:35:17 2011
@@ -12,6 +12,10 @@ $Id$
(No Changes)
+================== 3.2.0-dev ==================
+
+(No Changes)
+
================== 3.1.0-dev ==================
Upgrading from Solr 1.4
----------------------
@@ -80,12 +84,20 @@ Bug Fixes
* SOLR-2330: solrconfig.xml files in example-DIH are broken. (Matt Parker, koji)
+* SOLR-2116: Fix TikaConfig classloader bug in TikaEntityProcessor
+ (Martijn van Groningen via hossman)
+
+
Other Changes
----------------------
* SOLR-1821: Fix TimeZone-dependent test failure in TestEvaluatorBag.
(Chris Male via rmuir)
+* SOLR-2367: Reduced noise in test output by ensuring the properties file can be written.
+ (Gunnlaugur Thor Briem via rmuir)
+
+
Build
----------------------
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/build.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/build.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/build.xml Tue Mar 15 21:35:17 2011
@@ -276,7 +276,7 @@
<fileset dir="src/main/webapp" includes="**" />
</copy>
<mkdir dir="../../build/web/WEB-INF/lib"/>
- <copy file="target/${fullnamever}.jar" todir="${solr-path}/build/web/WEB-INF/lib"></copy>
+ <!--<copy file="target/${fullnamever}.jar" todir="${solr-path}/build/web/WEB-INF/lib"></copy>-->
<copy file="target/${fullnamever}.jar" todir="${solr-path}/dist"></copy>
<copy file="target/apache-${ant.project.name}-extras-${version}.jar" todir="${solr-path}/dist"></copy>
</target>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/main/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java Tue Mar 15 21:35:17 2011
@@ -17,15 +17,11 @@
package org.apache.solr.handler.dataimport;
import org.apache.commons.io.IOUtils;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImporter.COLUMN;
-import static org.apache.solr.handler.dataimport.XPathEntityProcessor.URL;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerDecorator;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -47,6 +43,11 @@ import java.io.StringWriter;
import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DataImporter.COLUMN;
+import static org.apache.solr.handler.dataimport.XPathEntityProcessor.URL;
/**
* <p>An implementation of {@link EntityProcessor} which reads data from rich docs
* using <a href="http://tika.apache.org/">Apache Tika</a>
@@ -65,19 +66,20 @@ public class TikaEntityProcessor extends
@Override
protected void firstInit(Context context) {
- String tikaConfigFile = context.getResolvedEntityAttribute("tikaConfig");
- if (tikaConfigFile == null) {
- tikaConfig = TikaConfig.getDefaultConfig();
- } else {
- File configFile = new File(tikaConfigFile);
- if (!configFile.isAbsolute()) {
- configFile = new File(context.getSolrCore().getResourceLoader().getConfigDir(), tikaConfigFile);
- }
- try {
+ try {
+ String tikaConfigFile = context.getResolvedEntityAttribute("tikaConfig");
+ if (tikaConfigFile == null) {
+ ClassLoader classLoader = context.getSolrCore().getResourceLoader().getClassLoader();
+ tikaConfig = new TikaConfig(classLoader);
+ } else {
+ File configFile = new File(tikaConfigFile);
+ if (!configFile.isAbsolute()) {
+ configFile = new File(context.getSolrCore().getResourceLoader().getConfigDir(), tikaConfigFile);
+ }
tikaConfig = new TikaConfig(configFile);
- } catch (Exception e) {
- wrapAndThrow (SEVERE, e,"Unable to load Tika Config");
}
+ } catch (Exception e) {
+ wrapAndThrow (SEVERE, e,"Unable to load Tika Config");
}
format = context.getResolvedEntityAttribute("format");
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/java/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java Tue Mar 15 21:35:17 2011
@@ -17,6 +17,7 @@
package org.apache.solr.handler.dataimport;
import org.junit.BeforeClass;
+import org.junit.Test;
/**Testcase for TikaEntityProcessor
* @version $Id$
@@ -25,9 +26,10 @@ import org.junit.BeforeClass;
public class TestTikaEntityProcessor extends AbstractDataImportHandlerTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema-no-unique-key.xml", "solr-dihextras");
+ initCore("dataimport-solrconfig.xml", "dataimport-schema-no-unique-key.xml", getFile("solr-dihextras").getAbsolutePath());
}
+ @Test
public void testIndexingWithTikaEntityProcessor() throws Exception {
String conf =
"<dataConfig>" +
@@ -35,12 +37,17 @@ public class TestTikaEntityProcessor ext
" <document>" +
" <entity processor=\"TikaEntityProcessor\" url=\"" + getFile("solr-word.pdf").getAbsolutePath() + "\" >" +
" <field column=\"Author\" meta=\"true\" name=\"author\"/>" +
- " <field column=\"title\" meta=\"true\" name=\"docTitle\"/>" +
+ " <field column=\"title\" meta=\"true\" name=\"title\"/>" +
" <field column=\"text\"/>" +
" </entity>" +
" </document>" +
"</dataConfig>";
runFullImport(conf);
- assertQ(req("*:*"), "//*[@numFound='1']");
+ assertQ(req("*:*")
+ ,"//*[@numFound='1']"
+ ,"//str[@name='author'][.='Grant Ingersoll']"
+ ,"//str[@name='title'][.='solr-word']"
+ ,"//str[@name='text']"
+ );
}
}
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/resources/solr-dihextras/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/resources/solr-dihextras/conf/dataimport-solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/resources/solr-dihextras/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/extras/test/resources/solr-dihextras/conf/dataimport-solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -18,19 +18,11 @@
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
- <!-- Set this to 'false' if you want solr to continue working after it has
- encountered an severe configuration error. In a production environment,
- you may want solr to keep working even if one handler is mis-configured.
-
- You may also set this to false using by setting the system property:
- -Dsolr.abortOnConfigurationError=false
- -->
- <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<indexDefaults>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java Tue Mar 15 21:35:17 2011
@@ -27,6 +27,7 @@ import org.apache.solr.common.util.Conte
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.SystemIdResolver;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
@@ -41,8 +42,10 @@ import org.apache.solr.update.processor.
import org.apache.solr.util.plugin.SolrCoreAware;
import java.util.*;
+import java.io.StringReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.xml.sax.InputSource;
/**
* <p>
@@ -108,9 +111,9 @@ public class DataImportHandler extends R
String configLoc = (String) defaults.get("config");
if (configLoc != null && configLoc.length() != 0) {
processConfiguration(defaults);
-
- importer = new DataImporter(SolrWriter.getResourceAsString(core
- .getResourceLoader().openResource(configLoc)), core,
+ final InputSource is = new InputSource(core.getResourceLoader().openConfig(configLoc));
+ is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(configLoc));
+ importer = new DataImporter(is, core,
dataSources, coreScopeSession);
}
}
@@ -162,7 +165,7 @@ public class DataImportHandler extends R
if (requestParams.dataConfig != null) {
try {
processConfiguration((NamedList) initArgs.get("defaults"));
- importer = new DataImporter(requestParams.dataConfig, req.getCore()
+ importer = new DataImporter(new InputSource(new StringReader(requestParams.dataConfig)), req.getCore()
, dataSources, coreScopeSession);
} catch (RuntimeException e) {
rsp.add("exception", DebugLogger.getStacktraceString(e));
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Tue Mar 15 21:35:17 2011
@@ -17,12 +17,15 @@
package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.SystemIdResolver;
+import org.apache.solr.common.util.XMLErrorLogger;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
@@ -32,6 +35,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
+import org.apache.commons.io.IOUtils;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -57,6 +61,7 @@ public class DataImporter {
}
private static final Logger LOG = LoggerFactory.getLogger(DataImporter.class);
+ private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(LOG);
private Status status = Status.IDLE;
@@ -87,7 +92,7 @@ public class DataImporter {
coreScopeSession = new ConcurrentHashMap<String, Object>();
}
- DataImporter(String dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session) {
+ DataImporter(InputSource dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session) {
if (dataConfig == null)
throw new DataImportHandlerException(SEVERE,
"Configuration not found");
@@ -140,7 +145,7 @@ public class DataImporter {
* Used by tests
*/
void loadAndInit(String configStr) {
- loadDataConfig(configStr);
+ loadDataConfig(new InputSource(new StringReader(configStr)));
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
for (DataConfig.Entity entity : config.document.entities) {
initEntity(entity, fields, false);
@@ -167,19 +172,32 @@ public class DataImporter {
}
- private void loadDataConfig(String configFile) {
+ private void loadDataConfig(InputSource configFile) {
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- try {
- dbf.setXIncludeAware(true);
- dbf.setNamespaceAware(true);
- } catch( UnsupportedOperationException e ) {
- LOG.warn( "XML parser doesn't support XInclude option" );
+
+ // only enable xinclude, if a a SolrCore and SystemId is present (makes no sense otherwise)
+ if (core != null && configFile.getSystemId() != null) {
+ try {
+ dbf.setXIncludeAware(true);
+ dbf.setNamespaceAware(true);
+ } catch( UnsupportedOperationException e ) {
+ LOG.warn( "XML parser doesn't support XInclude option" );
+ }
}
+
DocumentBuilder builder = dbf.newDocumentBuilder();
- Document document = builder.parse(new InputSource(new StringReader(
- configFile)));
+ if (core != null)
+ builder.setEntityResolver(new SystemIdResolver(core.getResourceLoader()));
+ builder.setErrorHandler(XMLLOG);
+ Document document;
+ try {
+ document = builder.parse(configFile);
+ } finally {
+ // some XML parsers are broken and don't close the byte stream (but they should according to spec)
+ IOUtils.closeQuietly(configFile.getByteStream());
+ }
config = new DataConfig();
NodeList elems = document.getElementsByTagName("dataConfig");
@@ -336,7 +354,7 @@ public class DataImporter {
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (Throwable t) {
- LOG.error("Full Import failed", t);
+ SolrException.log(LOG, "Full Import failed", t);
docBuilder.rollback();
} finally {
setStatus(Status.IDLE);
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue Mar 15 21:35:17 2011
@@ -17,6 +17,7 @@
package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.SolrCore;
import static org.apache.solr.handler.dataimport.SolrWriter.LAST_INDEX_KEY;
@@ -318,7 +319,11 @@ public class DocBuilder {
String keyName = root.isDocRoot ? root.getPk() : root.getSchemaPk();
Object key = map.get(keyName);
if(key == null) {
- LOG.warn("no key was available for deleteted pk query. keyName = " + keyName);
+ keyName = findMatchingPkColumn(keyName, map);
+ key = map.get(keyName);
+ }
+ if(key == null) {
+ LOG.warn("no key was available for deleted pk query. keyName = " + keyName);
continue;
}
writer.deleteDoc(key);
@@ -486,7 +491,7 @@ public class DocBuilder {
importStatistics.skipDocCount.getAndIncrement();
exception = null;//should not propogate up
} else {
- LOG.error("Exception while processing: "
+ SolrException.log(LOG, "Exception while processing: "
+ entity.name + " document : " + docWrapper, dihe);
}
if (dihe.getErrCode() == DataImportHandlerException.SEVERE)
@@ -645,7 +650,7 @@ public class DocBuilder {
importStatistics.skipDocCount.getAndIncrement();
doc = null;
} else {
- LOG.error("Exception while processing: "
+ SolrException.log(LOG, "Exception while processing: "
+ entity.name + " document : " + doc, e);
}
if (e.getErrCode() == DataImportHandlerException.SEVERE)
@@ -816,6 +821,28 @@ public class DocBuilder {
return entity.processor = new EntityProcessorWrapper(entityProcessor, this);
}
+ private String findMatchingPkColumn(String pk, Map<String, Object> row) {
+ if (row.containsKey(pk))
+ throw new IllegalArgumentException(
+ String.format("deltaQuery returned a row with null for primary key %s", pk));
+ String resolvedPk = null;
+ for (String columnName : row.keySet()) {
+ if (columnName.endsWith("." + pk) || pk.endsWith("." + columnName)) {
+ if (resolvedPk != null)
+ throw new IllegalArgumentException(
+ String.format(
+ "deltaQuery has more than one column (%s and %s) that might resolve to declared primary key pk='%s'",
+ resolvedPk, columnName, pk));
+ resolvedPk = columnName;
+ }
+ }
+ if (resolvedPk == null)
+ throw new IllegalArgumentException(
+ String.format("deltaQuery has no column to resolve to declared primary key pk='%s'", pk));
+ LOG.info(String.format("Resolving deltaQuery column '%s' to match entity's declared pk '%s'", resolvedPk, pk));
+ return resolvedPk;
+ }
+
/**
* <p> Collects unique keys of all Solr documents for whom one or more source tables have been changed since the last
* indexed time. </p> <p> Note: In our definition, unique key of Solr document is the primary key of the top level
@@ -852,13 +879,20 @@ public class DocBuilder {
Map<String, Map<String, Object>> deltaSet = new HashMap<String, Map<String, Object>>();
LOG.info("Running ModifiedRowKey() for Entity: " + entity.name);
//get the modified rows in this entity
+ String pk = entity.getPk();
while (true) {
Map<String, Object> row = entityProcessor.nextModifiedRowKey();
if (row == null)
break;
- deltaSet.put(row.get(entity.getPk()).toString(), row);
+ Object pkValue = row.get(pk);
+ if (pkValue == null) {
+ pk = findMatchingPkColumn(pk, row);
+ pkValue = row.get(pk);
+ }
+
+ deltaSet.put(pkValue.toString(), row);
importStatistics.rowsCount.incrementAndGet();
// check for abort
if (stop.get())
@@ -873,8 +907,14 @@ public class DocBuilder {
deletedSet.add(row);
+ Object pkValue = row.get(pk);
+ if (pkValue == null) {
+ pk = findMatchingPkColumn(pk, row);
+ pkValue = row.get(pk);
+ }
+
// Remove deleted rows from the delta rows
- String deletedRowPk = row.get(entity.getPk()).toString();
+ String deletedRowPk = pkValue.toString();
if (deltaSet.containsKey(deletedRowPk)) {
deltaSet.remove(deletedRowPk);
}
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java Tue Mar 15 21:35:17 2011
@@ -16,6 +16,7 @@
*/
package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.SolrException;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,7 +80,7 @@ public class EntityProcessorBase extends
rowIterator = null;
return null;
} catch (Exception e) {
- log.error("getNext() failed for query '" + query + "'", e);
+ SolrException.log(log, "getNext() failed for query '" + query + "'", e);
query = null;
rowIterator = null;
wrapAndThrow(DataImportHandlerException.WARN, e);
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java Tue Mar 15 21:35:17 2011
@@ -16,6 +16,7 @@
*/
package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.SolrException;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
import static org.apache.solr.handler.dataimport.EntityProcessorBase.*;
import static org.apache.solr.handler.dataimport.EntityProcessorBase.SKIP;
@@ -240,7 +241,7 @@ public class EntityProcessorWrapper exte
wrapAndThrow(SEVERE, e);
} else {
//SKIP is not really possible. If this calls the nextRow() again the Entityprocessor would be in an inconisttent state
- log.error("Exception in entity : "+ entityName, e);
+ SolrException.log(log, "Exception in entity : "+ entityName, e);
return null;
}
}
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java Tue Mar 15 21:35:17 2011
@@ -18,8 +18,13 @@ package org.apache.solr.handler.dataimpo
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.common.ResourceLoader;
+import org.apache.solr.common.util.SystemIdResolver;
+import org.apache.solr.common.util.XMLErrorLogger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.io.IOUtils;
import javax.xml.transform.Source;
import javax.xml.transform.TransformerException;
@@ -50,6 +55,7 @@ import java.util.concurrent.atomic.Atomi
*/
public class XPathEntityProcessor extends EntityProcessorBase {
private static final Logger LOG = LoggerFactory.getLogger(XPathEntityProcessor.class);
+ private static final XMLErrorLogger xmllog = new XMLErrorLogger(LOG);
private static final Map<String, Object> END_MARKER = new HashMap<String, Object>();
@@ -107,12 +113,27 @@ public class XPathEntityProcessor extend
if (xslt != null) {
xslt = context.replaceTokens(xslt);
try {
- Source xsltSource = new StreamSource(xslt);
// create an instance of TransformerFactory
TransformerFactory transFact = TransformerFactory.newInstance();
- xslTransformer = transFact.newTransformer(xsltSource);
- LOG
- .info("Using xslTransformer: "
+ final SolrCore core = context.getSolrCore();
+ final StreamSource xsltSource;
+ if (core != null) {
+ final ResourceLoader loader = core.getResourceLoader();
+ transFact.setURIResolver(new SystemIdResolver(loader).asURIResolver());
+ xsltSource = new StreamSource(loader.openResource(xslt),
+ SystemIdResolver.createSystemIdFromResourceName(xslt));
+ } else {
+ // fallback for tests
+ xsltSource = new StreamSource(xslt);
+ }
+ transFact.setErrorListener(xmllog);
+ try {
+ xslTransformer = transFact.newTransformer(xsltSource);
+ } finally {
+ // some XML parsers are broken and don't close the byte stream (but they should according to spec)
+ IOUtils.closeQuietly(xsltSource.getInputStream());
+ }
+ LOG.info("Using xslTransformer: "
+ xslTransformer.getClass().getName());
} catch (Exception e) {
throw new DataImportHandlerException(SEVERE,
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java Tue Mar 15 21:35:17 2011
@@ -16,6 +16,7 @@
*/
package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.util.XMLErrorLogger;
import javax.xml.stream.XMLInputFactory;
import static javax.xml.stream.XMLStreamConstants.*;
import javax.xml.stream.XMLStreamException;
@@ -25,6 +26,8 @@ import java.io.Reader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <p>
@@ -54,6 +57,9 @@ import java.util.regex.Pattern;
* @since solr 1.3
*/
public class XPathRecordReader {
+ private static final Logger LOG = LoggerFactory.getLogger(XPathRecordReader.class);
+ private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(LOG);
+
private Node rootNode = new Node("/", null);
/**
@@ -622,6 +628,7 @@ public class XPathRecordReader {
static{
factory.setProperty(XMLInputFactory.IS_VALIDATING , Boolean.FALSE);
factory.setProperty(XMLInputFactory.SUPPORT_DTD , Boolean.FALSE);
+ factory.setXMLReporter(XMLLOG);
}
/**Implement this interface to stream records as and when one is found.
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/webapp/admin/dataimport.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/webapp/admin/dataimport.jsp?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/webapp/admin/dataimport.jsp (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/main/webapp/admin/dataimport.jsp Tue Mar 15 21:35:17 2011
@@ -1,6 +1,5 @@
<%@ page import="org.apache.solr.request.SolrRequestHandler" %>
<%@ page import="java.util.Map" %>
-<%@ page import="org.apache.solr.handler.dataimport.DataImportHandler" %>
<%@ page contentType="text/html; charset=utf-8" pageEncoding="UTF-8"%>
<%--
Licensed to the Apache Software Foundation (ASF) under one or more
@@ -36,7 +35,7 @@ Select handler:
<ul>
<%
for (String key : handlers.keySet()) {
- if (handlers.get(key) instanceof DataImportHandler) { %>
+ if (handlers.get(key).getClass().getName().equals("org.apache.solr.handler.dataimport.DataImportHandler")) { %>
<li><a href="dataimport.jsp?handler=<%=key%>"><%=key%></a></li>
<%
}
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java Tue Mar 15 21:35:17 2011
@@ -54,7 +54,7 @@ public abstract class AbstractDataImport
// note, a little twisted that we shadow this static method
public static void initCore(String config, String schema) throws Exception {
- initCore(config, schema, "solr-dih");
+ initCore(config, schema, getFile("solr-dih").getAbsolutePath());
}
@Override
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java Tue Mar 15 21:35:17 2011
@@ -37,6 +37,7 @@ public class TestErrorHandling extends A
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ ignoreException("Unexpected close tag");
}
@Before @Override
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/contentstream-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/contentstream-solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/contentstream-solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/contentstream-solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -18,19 +18,11 @@
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
- <!-- Set this to 'false' if you want solr to continue working after it has
- encountered an severe configuration error. In a production environment,
- you may want solr to keep working even if one handler is mis-configured.
-
- You may also set this to false using by setting the system property:
- -Dsolr.abortOnConfigurationError=false
- -->
- <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<indexDefaults>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-nodatasource-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-nodatasource-solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-nodatasource-solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-nodatasource-solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -18,19 +18,11 @@
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
- <!-- Set this to 'false' if you want solr to continue working after it has
- encountered an severe configuration error. In a production environment,
- you may want solr to keep working even if one handler is mis-configured.
-
- You may also set this to false using by setting the system property:
- -Dsolr.abortOnConfigurationError=false
- -->
- <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<indexDefaults>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/dataimporthandler/src/test/resources/solr-dih/conf/dataimport-solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -18,19 +18,11 @@
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
- <!-- Set this to 'false' if you want solr to continue working after it has
- encountered an severe configuration error. In a production environment,
- you may want solr to keep working even if one handler is mis-configured.
-
- You may also set this to false using by setting the system property:
- -Dsolr.abortOnConfigurationError=false
- -->
- <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<indexDefaults>
Modified: lucene/dev/branches/bulkpostings/solr/contrib/extraction/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/extraction/CHANGES.txt?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/extraction/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/extraction/CHANGES.txt Tue Mar 15 21:35:17 2011
@@ -28,6 +28,10 @@ $Id:$
(No Changes)
+================== Release 3.2-dev ==================
+
+(No Changes)
+
================== Release 3.1-dev ==================
* SOLR-1902: Upgraded to Tika 0.8 and changed deprecated parse call
Modified: lucene/dev/branches/bulkpostings/solr/contrib/extraction/src/test/resources/solr-extraction/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/extraction/src/test/resources/solr-extraction/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/extraction/src/test/resources/solr-extraction/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/extraction/src/test/resources/solr-extraction/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -29,7 +29,7 @@
<!-- Used to specify an alternate directory to hold all index data.
It defaults to "index" if not present, and should probably
not be changed if replication is in use. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<indexDefaults>
<!-- Values here affect all index writers and act as a default
Modified: lucene/dev/branches/bulkpostings/solr/contrib/uima/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/uima/CHANGES.txt?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/uima/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/uima/CHANGES.txt Tue Mar 15 21:35:17 2011
@@ -11,7 +11,10 @@ Such fields could be language, concepts,
UIMA Dependency
---------------
-uima-core, OpenCalaisAnnotator, WhitespaceTokenizer, HMMTagger, AlchemyAPIAnnotator
-Current Version: 2.3.1-SNAPSHOT rev. 999276
+uimaj-core v2.3.1
+OpenCalaisAnnotator v2.3.1-SNAPSHOT rev. 1062868
+HMMTagger v2.3.1-SNAPSHOT rev. 1062868
+AlchemyAPIAnnotator v2.3.1-SNAPSHOT rev. 1062868
+WhitespaceTokenizer v2.3.1-SNAPSHOT rev. 1076132
$Id$
Modified: lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Tue Mar 15 21:35:17 2011
@@ -83,31 +83,26 @@ public class UIMAUpdateRequestProcessorT
+ " attached if you need it, but it is also committed to trunk and 3_x branch."
+ " Last Lucene European Conference has been held in Prague."));
assertU(commit());
- assertQ(req("language:english"), "//*[@numFound='1']");
+ assertQ(req("suggested_category:*"), "//*[@numFound='1']");
}
@Test
- public void testTwoUpdates() {
+ public void testTwoUpdates() throws Exception {
// this test requires an internet connection (e.g. opencalais api)
checkInternetConnection();
- try {
- addDoc(adoc("id", "1", "text", "The Apache Software Foundation is happy to announce "
- + "BarCampApache Sydney, Australia, the first ASF-backed event in the Southern "
- + "Hemisphere!"));
- assertU(commit());
- assertQ(req("language:english"), "//*[@numFound='1']");
-
- addDoc(adoc("id", "2", "text", "Taking place 11th December 2010 at the University "
- + "of Sydney's Darlington Centre, the BarCampApache \"unconference\" will be"
- + " attendee-driven, facilitated by members of the Apache community and will "
- + "focus on the Apache..."));
- assertU(commit());
- assertQ(req("language:english"), "//*[@numFound='2']");
+ addDoc(adoc("id", "1", "text", "The Apache Software Foundation is happy to announce "
+ + "BarCampApache Sydney, Australia, the first ASF-backed event in the Southern "
+ + "Hemisphere!"));
+ assertU(commit());
+ assertQ(req("suggested_category:*"), "//*[@numFound='1']");
- } catch (Exception e) {
- assumeNoException("Multiple updates on same instance didn't work", e);
- }
+ addDoc(adoc("id", "2", "text", "Taking place 11th December 2010 at the University "
+ + "of Sydney's Darlington Centre, the BarCampApache \"unconference\" will be"
+ + " attendee-driven, facilitated by members of the Apache community and will "
+ + "focus on the Apache..."));
+ assertU(commit());
+ assertQ(req("suggested_category:*"), "//*[@numFound='2']");
}
private void addDoc(String doc) throws Exception {
Modified: lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -24,15 +24,8 @@
https://issues.apache.org/jira/browse/SOLR-1167
-->
<config xmlns:xi="http://www.w3.org/2001/XInclude">
- <!--
- Set this to 'false' if you want solr to continue working after it
- has encountered an severe configuration error. In a production
- environment, you may want solr to keep working even if one handler
- is mis-configured. You may also set this to false using by setting
- the system property: -Dsolr.abortOnConfigurationError=false
- -->
+
<luceneMatchVersion>LUCENE_40</luceneMatchVersion>
- <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!--
lib directives can be used to instruct Solr to load an Jars
@@ -74,7 +67,7 @@
than the default ./data under the Solr home. If replication is in
use, this should match the replication configuration.
-->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
+ <dataDir>${solr.data.dir:}</dataDir>
<!--
@@ -1078,7 +1071,7 @@
<entities_apikey>04490000a72fe7ec5cb3497f14e77f338c86f2fe</entities_apikey>
<oc_licenseID>g6h9zamsdtwhb93nc247ecrs</oc_licenseID>
</runtimeParameters>
- <analysisEngine>/org/apache/uima/desc/OverridingParamsExtServicesAE.xml</analysisEngine>
+ <analysisEngine>/TestAE.xml</analysisEngine>
<analyzeFields merge="false">text</analyzeFields>
<fieldMapping>
<type name="org.apache.uima.alchemy.ts.categorization.Category">
Modified: lucene/dev/branches/bulkpostings/solr/example/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/README.txt?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/README.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/example/README.txt Tue Mar 15 21:35:17 2011
@@ -35,13 +35,15 @@ See also README.txt in the solr subdirec
http://wiki.apache.org/solr/SolrResources for a list of tutorials and
introductory articles.
-NOTE: This Solr example server references SolrCell jars outside of the server
-directory with <lib> statements in the solrconfig.xml. If you make a copy of
-this example server and wish to use the ExtractingRequestHandler (SolrCell),
+NOTE: This Solr example server references certain Solr jars outside of
+this server directory for non-core modules with <lib> statements in
+solrconfig.xml. If you make a copy of this example server and wish
+to use the ExtractingRequestHandler (SolrCell), DataImportHandler (DIH),
+UIMA, the clustering component, or other modules in "contrib",
you will need to copy the required jars into solr/lib or update the paths to
the jars in your solrconfig.xml.
-By default, start.jar starts Solr in Jetty using the default SolrHome
+By default, start.jar starts Solr in Jetty using the default solr home
directory of "./solr/" -- To run other example configurations, you can
speciy the solr.solr.home system property when starting jetty...
Modified: lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/db/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/db/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/db/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/db/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -28,6 +28,8 @@
<jmx />
+ <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-.*\.jar" />
+
<indexDefaults>
<!-- Values here affect all index writers and act as a default unless overridden. -->
<useCompoundFile>false</useCompoundFile>
Modified: lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/mail/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/mail/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/mail/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/mail/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -27,7 +27,7 @@
<luceneMatchVersion>LUCENE_40</luceneMatchVersion>
<lib dir="../../../../contrib/dataimporthandler/lib/" regex=".*jar$" />
- <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-extras-\d.*\.jar" />
+ <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-.*\.jar" />
<indexDefaults>
<!-- Values here affect all index writers and act as a default unless overridden. -->
Modified: lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/rss/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/rss/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/rss/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/rss/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -28,6 +28,8 @@
<jmx />
+ <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-.*\.jar" />
+
<indexDefaults>
<!-- Values here affect all index writers and act as a default unless overridden. -->
<useCompoundFile>false</useCompoundFile>
Modified: lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/tika/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/tika/conf/solrconfig.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/tika/conf/solrconfig.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/example/example-DIH/solr/tika/conf/solrconfig.xml Tue Mar 15 21:35:17 2011
@@ -27,13 +27,7 @@
<luceneMatchVersion>LUCENE_40</luceneMatchVersion>
<lib dir="../../../../contrib/extraction/lib" />
- <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-extras-\d.*\.jar" />
-
- <!-- Used to specify an alternate directory to hold all index data
- other than the default ./data under the Solr home.
- If replication is in use, this should match the replication configuration. -->
- <dataDir>${solr.data.dir:./solr/data}</dataDir>
-
+ <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-.*\.jar" />
<indexDefaults>
<!-- Values here affect all index writers and act as a default unless overridden. -->
Modified: lucene/dev/branches/bulkpostings/solr/example/exampledocs/test_utf8.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/exampledocs/test_utf8.sh?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/exampledocs/test_utf8.sh (original)
+++ lucene/dev/branches/bulkpostings/solr/example/exampledocs/test_utf8.sh Tue Mar 15 21:35:17 2011
@@ -53,12 +53,13 @@ else
fi
-#A codepoint outside of the BMP
-CODEPOINT='\\U00100058'
+#A unicode character outside of the BMP (a circle with an x inside)
+CHAR="ð"
+CODEPOINT='0x10308'
#URL encoded UTF8 of the codepoint
-URL_UTF8='%F4%80%81%98'
+URL_UTF8='%F0%90%8C%88'
#expected return of the python writer (currently uses UTF-16 surrogates)
-EXPECTED='\\udbc0\\udc58'
+EXPECTED='\\ud800\\udf08'
curl "$URL/select?q=$URL_UTF8&echoParams=explicit&wt=python" 2> /dev/null | grep $EXPECTED > /dev/null 2>&1
if [ $? = 0 ]; then
@@ -81,3 +82,12 @@ else
echo "ERROR: HTTP POST + URL params is not accepting UTF-8 beyond the basic multilingual plane"
fi
+#curl "$URL/select?q=$URL_UTF8&echoParams=explicit&wt=json" 2> /dev/null | od -tx1 -w1000 | sed 's/ //g' | grep 'f4808198' > /dev/null 2>&1
+curl "$URL/select?q=$URL_UTF8&echoParams=explicit&wt=json" 2> /dev/null | grep "$CHAR" > /dev/null 2>&1
+if [ $? = 0 ]; then
+ echo "Response correctly returns UTF-8 beyond the basic multilingual plane"
+else
+ echo "ERROR: Response can't return UTF-8 beyond the basic multilingual plane"
+fi
+
+
Modified: lucene/dev/branches/bulkpostings/solr/example/exampledocs/utf8-example.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/exampledocs/utf8-example.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/exampledocs/utf8-example.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/example/exampledocs/utf8-example.xml Tue Mar 15 21:35:17 2011
@@ -34,8 +34,8 @@
<field name="features">eaiou with umlauts: ëäïöü</field>
<field name="features">tag with escaped chars: <nicetag/></field>
<field name="features">escaped ampersand: Bonnie & Clyde</field>
+ <field name="features">Outside the BMP:ð codepoint=10308, a circle with an x inside. UTF8=f0908c88 UTF16=d800 df08</field>
<field name="price">0</field>
- <!-- no popularity, get the default from schema.xml -->
<field name="inStock">true</field>
</doc>
</add>
Modified: lucene/dev/branches/bulkpostings/solr/example/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/example/solr/conf/schema.xml?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/example/solr/conf/schema.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/example/solr/conf/schema.xml Tue Mar 15 21:35:17 2011
@@ -505,6 +505,7 @@
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
+ <dynamicField name="*_txt" type="text" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>