You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/05/30 09:53:46 UTC
svn commit: r1487777 [34/50] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/replicator/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/ma...
Modified: lucene/dev/branches/security/solr/cloud-dev/solrcloud-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/cloud-dev/solrcloud-start.sh?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/cloud-dev/solrcloud-start.sh (original)
+++ lucene/dev/branches/security/solr/cloud-dev/solrcloud-start.sh Thu May 30 07:53:18 2013
@@ -25,7 +25,7 @@ cp -r -f example example4
cp -r -f example example5
cp -r -f example example6
-java -classpath "example/solr-webapp/webapp/WEB-INF/lib/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost 127.0.0.1:9983 -solrhome example/solr -runzk 8983
+java -classpath "example/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost 127.0.0.1:9983 -solrhome example/solr -runzk 8983
cd example
java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example.log 2>&1 &
Modified: lucene/dev/branches/security/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/common-build.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/common-build.xml (original)
+++ lucene/dev/branches/security/solr/common-build.xml Thu May 30 07:53:18 2013
@@ -24,14 +24,10 @@
<dirname file="${ant.file.common-solr}" property="common-solr.dir"/>
<property name="Name" value="Solr" />
- <property name="version" value="5.0-SNAPSHOT"/>
- <property name="fullname" value="${ant.project.name}"/>
- <property name="fullnamever" value="${fullname}-${version}"/>
- <property name="final.name" value="${fullnamever}"/>
- <!-- solr uses 1.6 -->
- <property name="javac.source" value="1.6"/>
- <property name="javac.target" value="1.6"/>
+ <!-- solr uses 1.7 -->
+ <property name="javac.source" value="1.7"/>
+ <property name="javac.target" value="1.7"/>
<property name="javac.args" value=""/>
<property name="dest" location="${common-solr.dir}/build" />
@@ -43,7 +39,6 @@
<property name="tests.userdir" location="src/test-files"/>
<property name="example" location="${common-solr.dir}/example" />
<property name="javadoc.dir" location="${dest}/docs"/>
- <property name="tests.loggingfile" location="${common-solr.dir}/testlogging.properties"/>
<property name="tests.cleanthreads.sysprop" value="perClass"/>
<property name="changes.target.dir" location="${dest}/docs/changes"/>
@@ -51,6 +46,9 @@
<import file="${common-solr.dir}/../lucene/module-build.xml"/>
+ <!-- backwards compatibility with existing targets/tasks; TODO: remove this! -->
+ <property name="fullnamever" value="${final.name}"/>
+
<path id="additional.dependencies">
<fileset dir="${common-solr.dir}/core/lib" excludes="${common.classpath.excludes}"/>
<fileset dir="${common-solr.dir}/solrj/lib" excludes="${common.classpath.excludes}"/>
@@ -59,16 +57,6 @@
<fileset dir="lib" excludes="${common.classpath.excludes}" erroronmissingdir="false"/>
</path>
- <!-- Solr Specification Version
-
- This will be used in the Manifest file, and therefore must
- match the pattern "digit+{.digit+}*"
-
- By default, this should be set to "X.Y.M.${dateversion}"
- where X.Y.M is the last version released (on this branch).
- -->
- <property name="solr.spec.version" value="5.0.0.${dateversion}" />
-
<!-- defined here to be able to make the forbidden-api checker correctly
reference it. 'ivy.xml' is also referencing this property. -->
<property name="commons-io.version" value="2.1" />
@@ -170,7 +158,7 @@
<attribute name="excludes" default="**/pom.xml,**/*.iml"/>
<attribute name="metainf.source.dir" default="${common-solr.dir}"/>
<attribute name="implementation.title" default="org.apache.solr"/>
- <attribute name="spec.version" default="${solr.spec.version}"/>
+ <attribute name="spec.version" default="${spec.version}"/>
<attribute name="manifest.file" default="${manifest.file}"/>
<element name="nested" optional="true" implicit="true"/>
<sequential>
Modified: lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java (original)
+++ lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java Thu May 30 07:53:18 2013
@@ -17,7 +17,10 @@ package org.apache.solr.handler.clusteri
* limitations under the License.
*/
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -278,8 +281,8 @@ public class CarrotClusteringEngine exte
attributeBuilder.stemmerFactory(LuceneCarrot2StemmerFactory.class);
}
- // Pass the schema to SolrStopwordsCarrot2LexicalDataFactory.
- initAttributes.put("solrIndexSchema", core.getSchema());
+ // Pass the schema (via the core) to SolrStopwordsCarrot2LexicalDataFactory.
+ initAttributes.put("solrCore", core);
// Customize Carrot2's resource lookup to first look for resources
// using Solr's resource loader. If that fails, try loading from the classpath.
@@ -303,7 +306,7 @@ public class CarrotClusteringEngine exte
ct.setContextClassLoader(prev);
}
- SchemaField uniqueField = core.getSchema().getUniqueKeyField();
+ SchemaField uniqueField = core.getLatestSchema().getUniqueKeyField();
if (uniqueField == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
CarrotClusteringEngine.class.getSimpleName() + " requires the schema to have a uniqueKeyField");
Modified: lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java (original)
+++ lucene/dev/branches/security/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java Thu May 30 07:53:18 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.util.T
import org.apache.lucene.analysis.commongrams.CommonGramsFilterFactory;
import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
-import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.core.SolrCore;
import org.carrot2.core.LanguageCode;
import org.carrot2.core.attribute.Init;
import org.carrot2.core.attribute.Processing;
@@ -59,8 +59,8 @@ public class SolrStopwordsCarrot2Lexical
@Init
@Input
- @Attribute(key = "solrIndexSchema")
- private IndexSchema schema;
+ @Attribute(key = "solrCore")
+ private SolrCore core;
@Processing
@Input
@@ -86,7 +86,7 @@ public class SolrStopwordsCarrot2Lexical
// No need to synchronize here, Carrot2 ensures that instances
// of this class are not used by multiple threads at a time.
if (!solrStopWords.containsKey(fieldName)) {
- final Analyzer fieldAnalyzer = schema.getFieldType(fieldName)
+ final Analyzer fieldAnalyzer = core.getLatestSchema().getFieldType(fieldName)
.getAnalyzer();
if (fieldAnalyzer instanceof TokenizerChain) {
final TokenFilterFactory[] filterFactories = ((TokenizerChain) fieldAnalyzer)
Modified: lucene/dev/branches/security/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/security/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml Thu May 30 07:53:18 2013
@@ -165,13 +165,10 @@
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<!-- Case insensitive stop word removal.
- enablePositionIncrements=true ensures that a 'gap' is left to
- allow for accurate phrase queries.
-->
<filter class="solr.StopFilterFactory"
ignoreCase="true"
words="stopwords.txt"
- enablePositionIncrements="true"
/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
@@ -243,7 +240,7 @@
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
@@ -265,9 +262,6 @@
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
- compressed: [false] if this field should be stored using gzip compression
- (this will only apply if the field type is compressable; among
- the standard field types, only TextField and StrField are)
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java Thu May 30 07:53:18 2013
@@ -22,6 +22,8 @@ import org.apache.tika.metadata.Metadata
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.html.HtmlMapper;
+import org.apache.tika.parser.html.IdentityHtmlMapper;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerDecorator;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -63,7 +65,7 @@ public class TikaEntityProcessor extends
private boolean done = false;
private String parser;
static final String AUTO_PARSER = "org.apache.tika.parser.AutoDetectParser";
-
+ private String htmlMapper;
@Override
protected void firstInit(Context context) {
@@ -88,6 +90,13 @@ public class TikaEntityProcessor extends
format = "text";
if (!"html".equals(format) && !"xml".equals(format) && !"text".equals(format)&& !"none".equals(format) )
throw new DataImportHandlerException(SEVERE, "'format' can be one of text|html|xml|none");
+
+ htmlMapper = context.getResolvedEntityAttribute("htmlMapper");
+ if (htmlMapper == null)
+ htmlMapper = "default";
+ if (!"default".equals(htmlMapper) && !"identity".equals(htmlMapper))
+ throw new DataImportHandlerException(SEVERE, "'htmlMapper', if present, must be 'default' or 'identity'");
+
parser = context.getResolvedEntityAttribute("parser");
if(parser == null) {
parser = AUTO_PARSER;
@@ -124,7 +133,11 @@ public class TikaEntityProcessor extends
tikaParser = context.getSolrCore().getResourceLoader().newInstance(parser, Parser.class);
}
try {
- tikaParser.parse(is, contentHandler, metadata , new ParseContext());
+ ParseContext context = new ParseContext();
+ if ("identity".equals(htmlMapper)){
+ context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
+ }
+ tikaParser.parse(is, contentHandler, metadata , context);
} catch (Exception e) {
wrapAndThrow(SEVERE, e, "Unable to read content");
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java Thu May 30 07:53:18 2013
@@ -65,7 +65,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top1 did not return 2 messages", swi.docs.size(), 2);
@@ -80,7 +80,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its children did not return 8 messages", swi.docs.size(), 8);
@@ -96,7 +96,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 5 messages", swi.docs.size(), 5);
@@ -112,7 +112,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
@@ -129,7 +129,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
@@ -145,7 +145,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java Thu May 30 07:53:18 2013
@@ -55,6 +55,21 @@ public class TestTikaEntityProcessor ext
,"//str[@name='text']"
};
+ private String[] testsHTMLDefault = {
+ "//*[@numFound='1']"
+ , "//str[@name='text'][contains(.,'Basic div')]"
+ , "//str[@name='text'][contains(.,'<h1>')]"
+ , "//str[@name='text'][not(contains(.,'<div>'))]" //default mapper lower-cases elements as it maps
+ , "//str[@name='text'][not(contains(.,'<DIV>'))]"
+ };
+
+ private String[] testsHTMLIdentity = {
+ "//*[@numFound='1']"
+ , "//str[@name='text'][contains(.,'Basic div')]"
+ , "//str[@name='text'][contains(.,'<h1>')]"
+ , "//str[@name='text'][contains(.,'<div>')]"
+ , "//str[@name='text'][contains(.,'class=\"classAttribute\"')]" //attributes are lower-cased
+ };
@BeforeClass
public static void beforeClass() throws Exception {
@@ -67,4 +82,36 @@ public class TestTikaEntityProcessor ext
assertQ(req("*:*"), tests );
}
+ @Test
+ public void testTikaHTMLMapperEmpty() throws Exception {
+ runFullImport(getConfigHTML(null));
+ assertQ(req("*:*"), testsHTMLDefault);
+ }
+
+ @Test
+ public void testTikaHTMLMapperDefault() throws Exception {
+ runFullImport(getConfigHTML("default"));
+ assertQ(req("*:*"), testsHTMLDefault);
+ }
+
+ @Test
+ public void testTikaHTMLMapperIdentity() throws Exception {
+ runFullImport(getConfigHTML("identity"));
+ assertQ(req("*:*"), testsHTMLIdentity);
+ }
+
+ private String getConfigHTML(String htmlMapper) {
+ return
+ "<dataConfig>" +
+ " <dataSource type='BinFileDataSource'/>" +
+ " <document>" +
+ " <entity name='Tika' format='xml' processor='TikaEntityProcessor' " +
+ " url='" + getFile("dihextras/structured.html").getAbsolutePath() + "' " +
+ ((htmlMapper == null) ? "" : (" htmlMapper='" + htmlMapper + "'")) + ">" +
+ " <field column='text'/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ }
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Thu May 30 07:53:18 2013
@@ -124,7 +124,7 @@ public class DataImportHandler extends R
}
SolrParams params = req.getParams();
NamedList defaultParams = (NamedList) initArgs.get("defaults");
- RequestInfo requestParams = new RequestInfo(getParamsMap(params), contentStream);
+ RequestInfo requestParams = new RequestInfo(req, getParamsMap(params), contentStream);
String command = requestParams.getCommand();
if (DataImporter.SHOW_CONF_CMD.equals(command)) {
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Thu May 30 07:53:18 2013
@@ -20,9 +20,7 @@ package org.apache.solr.handler.dataimpo
import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.SystemIdResolver;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.XMLErrorLogger;
import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
@@ -36,7 +34,6 @@ import static org.apache.solr.handler.da
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
import static org.apache.solr.handler.dataimport.DocBuilder.loadClass;
import static org.apache.solr.handler.dataimport.config.ConfigNameConstants.CLASS;
-import static org.apache.solr.handler.dataimport.config.ConfigNameConstants.NAME;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,8 +48,14 @@ import javax.xml.parsers.DocumentBuilder
import java.io.IOException;
import java.io.StringReader;
-import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@@ -86,7 +89,6 @@ public class DataImporter {
private ReentrantLock importLock = new ReentrantLock();
private boolean isDeltaImportSupported = false;
private final String handlerName;
- private Map<String, SchemaField> lowerNameVsSchemaField = new HashMap<String, SchemaField>();
/**
* Only for testing purposes
@@ -98,8 +100,7 @@ public class DataImporter {
DataImporter(SolrCore core, String handlerName) {
this.handlerName = handlerName;
this.core = core;
- this.schema = core.getSchema();
- loadSchemaFieldMap();
+ this.schema = core.getLatestSchema();
}
@@ -110,8 +111,13 @@ public class DataImporter {
if (importLock.tryLock()) {
boolean success = false;
try {
+ if (null != params.getRequest()) {
+ if (schema != params.getRequest().getSchema()) {
+ schema = params.getRequest().getSchema();
+ }
+ }
String dataConfigText = params.getDataConfig();
- String dataconfigFile = (String) params.getConfigFile();
+ String dataconfigFile = params.getConfigFile();
InputSource is = null;
if(dataConfigText!=null && dataConfigText.length()>0) {
is = new InputSource(new StringReader(dataConfigText));
@@ -161,31 +167,14 @@ public class DataImporter {
- private void loadSchemaFieldMap() {
- Map<String, SchemaField> modLnvsf = new HashMap<String, SchemaField>();
- for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
- modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
- }
- lowerNameVsSchemaField = Collections.unmodifiableMap(modLnvsf);
- }
-
- public SchemaField getSchemaField(String caseInsensitiveName) {
- SchemaField schemaField = null;
- if(schema!=null) {
- schemaField = schema.getFieldOrNull(caseInsensitiveName);
- }
- if (schemaField == null) {
- schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
- }
- return schemaField;
+ public String getHandlerName() {
+ return handlerName;
}
- public String getHandlerName() {
- return handlerName;
- }
-
-
-
+ public IndexSchema getSchema() {
+ return schema;
+ }
+
/**
* Used by tests
*/
@@ -581,11 +570,7 @@ public class DataImporter {
public static final String TOTAL_DOCS_SKIPPED = "Total Documents Skipped";
}
- public IndexSchema getSchema() {
- return schema;
- }
-
- SolrCore getCore() {
+ public SolrCore getCore() {
return core;
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Thu May 30 07:53:18 2013
@@ -26,7 +26,9 @@ import org.apache.solr.handler.dataimpor
import org.apache.solr.handler.dataimport.config.EntityField;
import static org.apache.solr.handler.dataimport.SolrWriter.LAST_INDEX_KEY;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,7 +132,7 @@ public class DocBuilder {
indexerNamespace.put(LAST_INDEX_TIME, EPOCH);
}
indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
- indexerNamespace.put("request", reqParams.getRawParams());
+ indexerNamespace.put("request", new HashMap<String,Object>(reqParams.getRawParams()));
for (Entity entity : dataImporter.getConfig().getEntities()) {
String key = entity.getName() + "." + SolrWriter.LAST_INDEX_KEY;
Object lastIndex = persistedProperties.get(key);
@@ -625,11 +627,12 @@ public class DocBuilder {
if (value == null) continue;
if (key.startsWith("$")) continue;
Set<EntityField> field = entity.getColNameVsField().get(key);
- if (field == null && dataImporter.getSchema() != null) {
+ IndexSchema schema = null == reqParams.getRequest() ? null : reqParams.getRequest().getSchema();
+ if (field == null && schema != null) {
// This can be a dynamic field or a field which does not have an entry in data-config ( an implicit field)
- SchemaField sf = dataImporter.getSchema().getFieldOrNull(key);
+ SchemaField sf = schema.getFieldOrNull(key);
if (sf == null) {
- sf = dataImporter.getSchemaField(key);
+ sf = config.getSchemaField(key);
}
if (sf != null) {
addFieldToDoc(entry.getValue(), sf.getName(), 1.0f, sf.multiValued(), doc);
@@ -643,7 +646,7 @@ public class DocBuilder {
boolean toWrite = f.isToWrite();
if(f.isDynamicName()){
name = vr.replaceTokens(name);
- SchemaField schemaField = dataImporter.getSchemaField(name);
+ SchemaField schemaField = config.getSchemaField(name);
if(schemaField == null) {
toWrite = false;
} else {
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java Thu May 30 07:53:18 2013
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrQueryRequest;
public class RequestInfo {
private final String command;
@@ -37,13 +38,15 @@ public class RequestInfo {
private final List<String> entitiesToRun;
private final Map<String,Object> rawParams;
private final String configFile;
- private final String dataConfig;
+ private final String dataConfig;
+ private final SolrQueryRequest request;
//TODO: find a different home for these two...
private final ContentStream contentStream;
private final DebugInfo debugInfo;
- public RequestInfo(Map<String,Object> requestParams, ContentStream stream) {
+ public RequestInfo(SolrQueryRequest request, Map<String,Object> requestParams, ContentStream stream) {
+ this.request = request;
this.contentStream = stream;
if (requestParams.containsKey("command")) {
command = (String) requestParams.get("command");
@@ -167,4 +170,8 @@ public class RequestInfo {
public String getConfigFile() {
return configFile;
}
+
+ public SolrQueryRequest getRequest() {
+ return request;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java Thu May 30 07:53:18 2013
@@ -91,8 +91,9 @@ public class VariableResolver {
Object r = null;
if (name != null) {
String[] nameParts = DOT_PATTERN.split(name);
- Map<String,Object> currentLevel = currentLevelMap(nameParts,
+ CurrentLevel cr = currentLevelMap(nameParts,
rootNamespace, false);
+ Map<String,Object> currentLevel = cr.map;
r = currentLevel.get(nameParts[nameParts.length - 1]);
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
&& name.length() > FUNCTIONS_NAMESPACE.length()) {
@@ -103,6 +104,16 @@ public class VariableResolver {
return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
}
if (r == null) {
+ StringBuilder sb = new StringBuilder();
+ for(int i=cr.level ; i<nameParts.length ; i++) {
+ if(sb.length()>0) {
+ sb.append(".");
+ }
+ sb.append(nameParts[i]);
+ }
+ r = cr.map.get(sb.toString());
+ }
+ if (r == null) {
r = System.getProperty(name);
}
}
@@ -181,28 +192,41 @@ public class VariableResolver {
if (name != null) {
String[] nameParts = DOT_PATTERN.split(name);
Map<String,Object> nameResolveLevel = currentLevelMap(nameParts,
- rootNamespace, false);
+ rootNamespace, false).map;
nameResolveLevel.put(nameParts[nameParts.length - 1], newMap);
} else {
for (Map.Entry<String,Object> entry : newMap.entrySet()) {
String[] keyParts = DOT_PATTERN.split(entry.getKey());
Map<String,Object> currentLevel = rootNamespace;
- currentLevel = currentLevelMap(keyParts, currentLevel, false);
+ currentLevel = currentLevelMap(keyParts, currentLevel, false).map;
currentLevel.put(keyParts[keyParts.length - 1], entry.getValue());
}
}
}
}
- private Map<String,Object> currentLevelMap(String[] keyParts,
+ class CurrentLevel {
+ final Map<String,Object> map;
+ final int level;
+ CurrentLevel(int level, Map<String,Object> map) {
+ this.level = level;
+ this.map = map;
+ }
+ }
+
+ private CurrentLevel currentLevelMap(String[] keyParts,
Map<String,Object> currentLevel, boolean includeLastLevel) {
int j = includeLastLevel ? keyParts.length : keyParts.length - 1;
for (int i = 0; i < j; i++) {
Object o = currentLevel.get(keyParts[i]);
if (o == null) {
- Map<String,Object> nextLevel = new HashMap<String,Object>();
- currentLevel.put(keyParts[i], nextLevel);
- currentLevel = nextLevel;
+ if(i == j-1) {
+ Map<String,Object> nextLevel = new HashMap<String,Object>();
+ currentLevel.put(keyParts[i], nextLevel);
+ currentLevel = nextLevel;
+ } else {
+ return new CurrentLevel(i, currentLevel);
+ }
} else if (o instanceof Map<?,?>) {
@SuppressWarnings("unchecked")
Map<String,Object> nextLevel = (Map<String,Object>) o;
@@ -212,7 +236,7 @@ public class VariableResolver {
"Non-leaf nodes should be of type java.util.Map");
}
}
- return currentLevel;
+ return new CurrentLevel(j-1, currentLevel);
}
public void removeNamespace(String name) {
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java Thu May 30 07:53:18 2013
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +33,6 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class ConfigParseUtil {
- private static final Logger LOG = LoggerFactory .getLogger(ConfigParseUtil.class);
public static String getStringAttribute(Element e, String name, String def) {
String r = e.getAttribute(name);
@@ -77,46 +77,4 @@ public class ConfigParseUtil {
}
return result;
}
-
- public static void verifyWithSchema(DataImporter di, Map<String,EntityField> fields) {
- Map<String,SchemaField> schemaFields = null;
- if (di.getSchema() == null) {
- schemaFields = Collections.emptyMap();
- } else {
- schemaFields = di.getSchema().getFields();
- }
- for (Map.Entry<String,SchemaField> entry : schemaFields.entrySet()) {
- SchemaField sf = entry.getValue();
- if (!fields.containsKey(sf.getName())) {
- if (sf.isRequired()) {
- LOG
- .info(sf.getName()
- + " is a required field in SolrSchema . But not found in DataConfig");
- }
- }
- }
- for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
- EntityField fld = entry.getValue();
- SchemaField field = di.getSchemaField(fld.getName());
- if (field == null) {
- LOG
- .info("The field :"
- + fld.getName()
- + " present in DataConfig does not have a counterpart in Solr Schema");
- }
- }
- }
-
- public static Map<String,EntityField> gatherAllFields(DataImporter di, Entity e) {
- Map<String,EntityField> fields = new HashMap<String,EntityField>();
- if (e.getFields() != null) {
- for (EntityField f : e.getFields()) {
- fields.put(f.getName(), f);
- }
- }
- for (Entity e1 : e.getChildren()) {
- fields.putAll(gatherAllFields(di, e1));
- }
- return fields;
- }
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java Thu May 30 07:53:18 2013
@@ -2,10 +2,16 @@ package org.apache.solr.handler.dataimpo
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
/*
@@ -41,6 +47,8 @@ import org.w3c.dom.Element;
* @since solr 1.3
*/
public class DIHConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(DIHConfiguration.class);
+
// TODO - remove from here and add it to entity
private final String deleteQuery;
@@ -51,10 +59,14 @@ public class DIHConfiguration {
private final Script script;
private final Map<String, Map<String,String>> dataSources;
private final PropertyWriter propertyWriter;
+ private final IndexSchema schema;
+ private final Map<String,SchemaField> lowerNameVsSchemaField;
public DIHConfiguration(Element element, DataImporter di,
List<Map<String,String>> functions, Script script,
Map<String,Map<String,String>> dataSources, PropertyWriter pw) {
+ schema = di.getSchema();
+ lowerNameVsSchemaField = null == schema ? Collections.<String,SchemaField>emptyMap() : loadSchemaFieldMap();
this.deleteQuery = ConfigParseUtil.getStringAttribute(element, "deleteQuery", null);
this.onImportStart = ConfigParseUtil.getStringAttribute(element, "onImportStart", null);
this.onImportEnd = ConfigParseUtil.getStringAttribute(element, "onImportEnd", null);
@@ -62,9 +74,9 @@ public class DIHConfiguration {
List<Element> l = ConfigParseUtil.getChildNodes(element, "entity");
boolean docRootFound = false;
for (Element e : l) {
- Entity entity = new Entity(docRootFound, e, di, null);
- Map<String, EntityField> fields = ConfigParseUtil.gatherAllFields(di, entity);
- ConfigParseUtil.verifyWithSchema(di, fields);
+ Entity entity = new Entity(docRootFound, e, di, this, null);
+ Map<String, EntityField> fields = gatherAllFields(di, entity);
+ verifyWithSchema(fields);
modEntities.add(entity);
}
this.entities = Collections.unmodifiableList(modEntities);
@@ -80,6 +92,64 @@ public class DIHConfiguration {
this.dataSources = Collections.unmodifiableMap(dataSources);
this.propertyWriter = pw;
}
+
+ private void verifyWithSchema(Map<String,EntityField> fields) {
+ Map<String,SchemaField> schemaFields = null;
+ if (schema == null) {
+ schemaFields = Collections.emptyMap();
+ } else {
+ schemaFields = schema.getFields();
+ }
+ for (Map.Entry<String,SchemaField> entry : schemaFields.entrySet()) {
+ SchemaField sf = entry.getValue();
+ if (!fields.containsKey(sf.getName())) {
+ if (sf.isRequired()) {
+ LOG.info(sf.getName() + " is a required field in SolrSchema . But not found in DataConfig");
+ }
+ }
+ }
+ for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
+ EntityField fld = entry.getValue();
+ SchemaField field = getSchemaField(fld.getName());
+ if (field == null) {
+ LOG.info("The field :" + fld.getName() + " present in DataConfig does not have a counterpart in Solr Schema");
+ }
+ }
+ }
+
+ private Map<String,EntityField> gatherAllFields(DataImporter di, Entity e) {
+ Map<String,EntityField> fields = new HashMap<String,EntityField>();
+ if (e.getFields() != null) {
+ for (EntityField f : e.getFields()) {
+ fields.put(f.getName(), f);
+ }
+ }
+ for (Entity e1 : e.getChildren()) {
+ fields.putAll(gatherAllFields(di, e1));
+ }
+ return fields;
+ }
+
+ private Map<String,SchemaField> loadSchemaFieldMap() {
+ Map<String, SchemaField> modLnvsf = new HashMap<String, SchemaField>();
+ for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
+ modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
+ }
+ return Collections.unmodifiableMap(modLnvsf);
+ }
+
+ public SchemaField getSchemaField(String caseInsensitiveName) {
+ SchemaField schemaField = null;
+ if(schema!=null) {
+ schemaField = schema.getFieldOrNull(caseInsensitiveName);
+ }
+ if (schemaField == null) {
+ schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
+ }
+ return schemaField;
+ }
+
+
public String getDeleteQuery() {
return deleteQuery;
}
@@ -104,4 +174,8 @@ public class DIHConfiguration {
public PropertyWriter getPropertyWriter() {
return propertyWriter;
}
+
+ public IndexSchema getSchema() {
+ return schema;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java Thu May 30 07:53:18 2013
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.solr.handler.dataimport.DataImportHandlerException;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.w3c.dom.Element;
@@ -45,9 +46,11 @@ public class Entity {
private final Map<String,Set<EntityField>> colNameVsField;
private final Map<String,String> allAttributes;
private final List<Map<String,String>> allFieldAttributes;
+ private final DIHConfiguration config;
- public Entity(boolean docRootFound, Element element, DataImporter di, Entity parent) {
+ public Entity(boolean docRootFound, Element element, DataImporter di, DIHConfiguration config, Entity parent) {
this.parentEntity = parent;
+ this.config = config;
String modName = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.NAME, null);
if (modName == null) {
@@ -85,13 +88,13 @@ public class Entity {
List<Map<String,String>> modAllFieldAttributes = new ArrayList<Map<String,String>>();
for (Element elem : n) {
EntityField.Builder fieldBuilder = new EntityField.Builder(elem);
- if (di.getSchema() != null) {
+ if (config.getSchema() != null) {
if (fieldBuilder.getNameOrColumn() != null
&& fieldBuilder.getNameOrColumn().contains("${")) {
fieldBuilder.dynamicName = true;
} else {
- SchemaField schemaField = di.getSchemaField(fieldBuilder
- .getNameOrColumn());
+ SchemaField schemaField = config.getSchemaField
+ (fieldBuilder.getNameOrColumn());
if (schemaField != null) {
fieldBuilder.name = schemaField.getName();
fieldBuilder.multiValued = schemaField.multiValued();
@@ -139,8 +142,8 @@ public class Entity {
.unmodifiableList(modAllFieldAttributes);
String modPkMappingFromSchema = null;
- if (di.getSchema() != null) {
- SchemaField uniqueKey = di.getSchema().getUniqueKeyField();
+ if (config.getSchema() != null) {
+ SchemaField uniqueKey = config.getSchema().getUniqueKeyField();
if (uniqueKey != null) {
modPkMappingFromSchema = uniqueKey.getName();
// if no fields are mentioned . solr uniqueKey is same as dih 'pk'
@@ -160,8 +163,7 @@ public class Entity {
n = ConfigParseUtil.getChildNodes(element, "entity");
List<Entity> modEntities = new ArrayList<Entity>();
for (Element elem : n) {
- modEntities
- .add(new Entity((docRootFound || this.docRoot), elem, di, this));
+ modEntities.add(new Entity((docRootFound || this.docRoot), elem, di, config, this));
}
this.children = Collections.unmodifiableList(modEntities);
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml Thu May 30 07:53:18 2013
@@ -224,7 +224,7 @@
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
@@ -246,9 +246,6 @@
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
- compressed: [false] if this field should be stored using gzip compression
- (this will only apply if the field type is compressable; among
- the standard field types, only TextField and StrField are)
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml Thu May 30 07:53:18 2013
@@ -239,6 +239,9 @@
</requestHandler>
<requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ <lst name="defaults">
+ <str name="dots.in.hsqldb.driver">org.hsqldb.jdbcDriver</str>
+ </lst>
</requestHandler>
<!--
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java Thu May 30 07:53:18 2013
@@ -37,7 +37,7 @@ public class TestContextImpl extends Abs
public void testCoreScope() {
DataImporter di = new DataImporter();
di.loadAndInit("<dataConfig><document /></dataConfig>");
- DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(new HashMap<String,Object>(), null));
+ DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(null, new HashMap<String,Object>(), null));
ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, db);
String lala = new String("lala");
ctx.setSessionAttribute("huhu", lala, Context.SCOPE_SOLR_CORE);
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java Thu May 30 07:53:18 2013
@@ -57,7 +57,7 @@ public class TestDocBuilder extends Abst
DIHConfiguration cfg = di.getConfig();
Entity ent = cfg.getEntities().get(0);
MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
@@ -77,7 +77,7 @@ public class TestDocBuilder extends Abst
Entity ent = cfg.getEntities().get(0);
MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "delta-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.FALSE, swi.deleteAllCalled);
@@ -98,7 +98,7 @@ public class TestDocBuilder extends Abst
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
@@ -127,7 +127,7 @@ public class TestDocBuilder extends Abst
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.FALSE, swi.deleteAllCalled);
@@ -153,7 +153,7 @@ public class TestDocBuilder extends Abst
di.loadAndInit(dc_singleEntity);
DIHConfiguration cfg = di.getConfig();
Entity ent = cfg.getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("id", 1, "desc", "one"));
l.add(createMap("id", 2, "desc", "two"));
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java Thu May 30 07:53:18 2013
@@ -36,7 +36,7 @@ public class TestFieldReader extends Abs
DataImporter di = new DataImporter();
di.loadAndInit(config);
TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("xml", xml));
MockDataSource.setIterator("select * from a", l.iterator());
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java Thu May 30 07:53:18 2013
@@ -34,7 +34,7 @@ public class TestPlainTextEntityProcesso
DataImporter di = new DataImporter();
di.loadAndInit(DATA_CONFIG);
TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
di.runCmd(rp, sw);
assertEquals(DS.s, sw.docs.get(0).getFieldValue("x"));
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java Thu May 30 07:53:18 2013
@@ -53,9 +53,9 @@ public class TestScriptTransformer exten
EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
sep.init(context);
sep.applyTransformer(map);
- assertEquals(map.get("name"), "Hello Scott");
+ assertEquals("Hello Scott", map.get("name").toString());
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
@@ -86,9 +86,9 @@ public class TestScriptTransformer exten
EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
sep.init(context);
sep.applyTransformer(map);
- assertEquals(map.get("name"), "Hello Scott");
+ assertEquals("Hello Scott", map.get("name").toString());
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
@@ -104,7 +104,7 @@ public class TestScriptTransformer exten
DIHConfiguration dc = di.readFromXml(document);
assertTrue(dc.getScript().getText().indexOf("checkNextToken") > -1);
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
@@ -131,7 +131,7 @@ public class TestScriptTransformer exten
sep.applyTransformer(map);
assertNull(map.get("$hasMore"));
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
Modified: lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java (original)
+++ lucene/dev/branches/security/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java Thu May 30 07:53:18 2013
@@ -62,7 +62,7 @@ public class TestVariableResolverEndToEn
String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault() + "'");
StringBuilder sb = new StringBuilder();
sb.append("<dataConfig> \n");
- sb.append("<dataSource name=\"hsqldb\" driver=\"org.hsqldb.jdbcDriver\" url=\"jdbc:hsqldb:mem:.\" /> \n");
+ sb.append("<dataSource name=\"hsqldb\" driver=\"${dataimporter.request.dots.in.hsqldb.driver}\" url=\"jdbc:hsqldb:mem:.\" /> \n");
sb.append("<document name=\"TestEvaluators\"> \n");
sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
sb.append(" query=\"" +
Modified: lucene/dev/branches/security/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/security/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Thu May 30 07:53:18 2013
@@ -28,10 +28,10 @@ import org.apache.solr.common.params.Upd
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.loader.ContentStreamLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.IndexSchema;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.tika.config.TikaConfig;
@@ -81,7 +81,7 @@ public class ExtractingDocumentLoader ex
private static final XPathParser PARSER =
new XPathParser("xhtml", XHTMLContentHandler.XHTML);
- final IndexSchema schema;
+ final SolrCore core;
final SolrParams params;
final UpdateRequestProcessor processor;
final boolean ignoreTikaException;
@@ -95,7 +95,7 @@ public class ExtractingDocumentLoader ex
public ExtractingDocumentLoader(SolrQueryRequest req, UpdateRequestProcessor processor,
TikaConfig config, SolrContentHandlerFactory factory) {
this.params = req.getParams();
- schema = req.getSchema();
+ this.core = req.getCore();
this.config = config;
this.processor = processor;
@@ -167,7 +167,7 @@ public class ExtractingDocumentLoader ex
String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);
boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false);
- SolrContentHandler handler = factory.createSolrContentHandler(metadata, params, schema);
+ SolrContentHandler handler = factory.createSolrContentHandler(metadata, params, req.getSchema());
ContentHandler parsingHandler = handler;
StringWriter writer = null;
Modified: lucene/dev/branches/security/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/security/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml Thu May 30 07:53:18 2013
@@ -285,7 +285,7 @@
<fieldtype name="syn" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory"/>
- <filter name="syn" class="solr.SynonymFilterFactory" synonyms="synonyms.txt"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"/>
</analyzer>
</fieldtype>
@@ -354,8 +354,8 @@
termPositions="true" termOffsets="true"/>
<!-- test highlit field settings -->
- <field name="test_hlt" type="highlittext" indexed="true" compressed="true"/>
- <field name="test_hlt_off" type="highlittext" indexed="true" compressed="false"/>
+ <field name="test_hlt" type="highlittext" indexed="true"/>
+ <field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
<field name="teststop" type="teststop" indexed="true" stored="true"/>
Modified: lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java (original)
+++ lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java Thu May 30 07:53:18 2013
@@ -17,8 +17,6 @@ package org.apache.solr.uima.processor;
* limitations under the License.
*/
-import java.util.Map;
-
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
import org.apache.uima.cas.FSIterator;
@@ -29,6 +27,8 @@ import org.apache.uima.jcas.tcas.Annotat
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Map;
+
/**
* Map UIMA types and features over fields of a Solr document
*
@@ -64,16 +64,18 @@ public class UIMAToSolrMapper {
String fieldNameFeatureValue = fieldNameFeature == null ? null :
fs.getFeatureValueAsString(type.getFeatureByBaseName(fieldNameFeature));
String fieldName = mapField.getFieldName(fieldNameFeatureValue);
- log.info(new StringBuilder("mapping ").append(typeName).append("@").append(featureName)
- .append(" to ").append(fieldName).toString());
+ if (log.isInfoEnabled()) {
+ log.info("mapping {}@{} to {}", new Object[]{typeName, featureName, fieldName});
+ }
String featureValue;
if (fs instanceof Annotation && "coveredText".equals(featureName)) {
featureValue = ((Annotation) fs).getCoveredText();
} else {
featureValue = fs.getFeatureValueAsString(type.getFeatureByBaseName(featureName));
}
- log.info(new StringBuilder("writing ").append(featureValue).append(" in ").append(
- fieldName).toString());
+ if (log.isDebugEnabled()) {
+ log.debug("writing {} in {}", new Object[]{featureValue, fieldName});
+ }
document.addField(fieldName, featureValue, 1.0f);
}
}
Modified: lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java (original)
+++ lucene/dev/branches/security/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java Thu May 30 07:53:18 2013
@@ -92,7 +92,7 @@ public class UIMAUpdateRequestProcessor
} catch (Exception e) {
String logField = solrUIMAConfiguration.getLogField();
if(logField == null){
- SchemaField uniqueKeyField = solrCore.getSchema().getUniqueKeyField();
+ SchemaField uniqueKeyField = cmd.getReq().getSchema().getUniqueKeyField();
if(uniqueKeyField != null){
logField = uniqueKeyField.getName();
}
@@ -111,9 +111,9 @@ public class UIMAUpdateRequestProcessor
debugString = " null text";
}
if (solrUIMAConfiguration.isIgnoreErrors()) {
- log.warn(new StringBuilder("skip the text processing due to ")
+ log.warn("skip the text processing due to {}",new StringBuilder()
.append(e.getLocalizedMessage()).append(optionalFieldInfo)
- .append(debugString).toString());
+ .append(debugString));
} else {
throw new SolrException(ErrorCode.SERVER_ERROR,
new StringBuilder("processing error ")
@@ -150,7 +150,9 @@ public class UIMAUpdateRequestProcessor
/* process a field value executing UIMA the CAS containing it as document text */
private JCas processText(String textFieldValue) throws ResourceInitializationException,
AnalysisEngineProcessException {
- log.info(new StringBuilder("Analyzing text").toString());
+ if (log.isDebugEnabled()) {
+ log.debug("Analyzing text");
+ }
/* get the UIMA analysis engine */
AnalysisEngine ae = aeProvider.getAE();
@@ -160,7 +162,9 @@ public class UIMAUpdateRequestProcessor
/* perform analysis on text field */
ae.process(jcas);
- log.info("Text processing completed");
+ if (log.isDebugEnabled()) {
+ log.debug("Text processing completed");
+ }
return jcas;
}
Modified: lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml Thu May 30 07:53:18 2013
@@ -253,11 +253,9 @@
-->
<!--
Case insensitive stop word removal. add
- enablePositionIncrements=true in both the index and query
- analyzers to leave a 'gap' for more accurate phrase queries.
-->
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
catenateNumbers="1" catenateAll="0" splitOnCaseChange="1" />
@@ -269,7 +267,7 @@
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"
ignoreCase="true" expand="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="0"
catenateNumbers="0" catenateAll="0" splitOnCaseChange="1" />
@@ -316,7 +314,7 @@
<analyzer type="index">
<tokenizer class="solr.MockTokenizerFactory" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
catenateNumbers="1" catenateAll="0" splitOnCaseChange="0" />
@@ -327,7 +325,7 @@
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"
ignoreCase="true" expand="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="0"
catenateNumbers="0" catenateAll="0" splitOnCaseChange="0" />
@@ -346,7 +344,7 @@
<analyzer type="index">
<tokenizer class="solr.MockTokenizerFactory" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
catenateNumbers="1" catenateAll="0" splitOnCaseChange="0" />
@@ -360,7 +358,7 @@
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"
ignoreCase="true" expand="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="0"
catenateNumbers="0" catenateAll="0" splitOnCaseChange="0" />
@@ -406,7 +404,7 @@
documentation for more information on pattern and replacement
string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory" pattern="([^a-z])"
replacement="" replace="all" />
@@ -470,10 +468,7 @@
field type: mandatory - the name of a previously defined type from
the <types> section indexed: true if this field should be indexed
(searchable or sortable) stored: true if this field should be
- retrievable compressed: [false] if this field should be stored
- using gzip compression (this will only apply if the field type is
- compressable; among the standard field types, only TextField and
- StrField are) multiValued: true if this field may contain multiple
+ retrievable multiValued: true if this field may contain multiple
values per document omitNorms: (expert) set to true to omit the
norms associated with this field (this disables length
normalization and index-time boosting for the field, and saves
Modified: lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml (original)
+++ lucene/dev/branches/security/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml Thu May 30 07:53:18 2013
@@ -253,8 +253,6 @@
-->
<!--
Case insensitive stop word removal. add
- enablePositionIncrements=true in both the index and query
- analyzers to leave a 'gap' for more accurate phrase queries.
-->
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
@@ -402,7 +400,7 @@
documentation for more information on pattern and replacement
string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory" pattern="([^a-z])"
replacement="" replace="all" />
@@ -466,10 +464,7 @@
field type: mandatory - the name of a previously defined type from
the <types> section indexed: true if this field should be indexed
(searchable or sortable) stored: true if this field should be
- retrievable compressed: [false] if this field should be stored
- using gzip compression (this will only apply if the field type is
- compressable; among the standard field types, only TextField and
- StrField are) multiValued: true if this field may contain multiple
+ retrievable multiValued: true if this field may contain multiple
values per document omitNorms: (expert) set to true to omit the
norms associated with this field (this disables length
normalization and index-time boosting for the field, and saves
Modified: lucene/dev/branches/security/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java (original)
+++ lucene/dev/branches/security/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java Thu May 30 07:53:18 2013
@@ -19,6 +19,7 @@ package org.apache.solr.uima.analysis;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.IndexSchema;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,10 +35,11 @@ public class UIMATokenizersSolrIntegrati
@Test
public void testInitialization() throws Exception {
- assertNotNull(h.getCore().getSchema().getField("sentences"));
- assertNotNull(h.getCore().getSchema().getFieldType("sentences"));
- assertNotNull(h.getCore().getSchema().getField("nouns"));
- assertNotNull(h.getCore().getSchema().getFieldType("nouns"));
+ IndexSchema schema = h.getCore().getLatestSchema();
+ assertNotNull(schema.getField("sentences"));
+ assertNotNull(schema.getFieldType("sentences"));
+ assertNotNull(schema.getField("nouns"));
+ assertNotNull(schema.getFieldType("nouns"));
}
@Test
Modified: lucene/dev/branches/security/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/ivy.xml?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/ivy.xml (original)
+++ lucene/dev/branches/security/solr/core/ivy.xml Thu May 30 07:53:18 2013
@@ -24,7 +24,7 @@
<dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
<dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
<dependency org="commons-lang" name="commons-lang" rev="2.6" transitive="false"/>
- <dependency org="com.google.guava" name="guava" rev="13.0.1" transitive="false"/>
+ <dependency org="com.google.guava" name="guava" rev="14.0.1" transitive="false"/>
<dependency org="org.easymock" name="easymock" rev="3.0" transitive="false"/>
<dependency org="cglib" name="cglib-nodep" rev="2.2" transitive="false"/>
<dependency org="org.objenesis" name="objenesis" rev="1.2" transitive="false"/>
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java Thu May 30 07:53:18 2013
@@ -19,6 +19,7 @@ package org.apache.solr.analysis;
*/
import java.io.Reader;
+import java.util.Map;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -54,6 +55,14 @@ import org.apache.lucene.analysis.util.C
@Deprecated
public class LegacyHTMLStripCharFilterFactory extends CharFilterFactory {
+ /** Creates a new LegacyHTMLStripCharFilterFactory */
+ public LegacyHTMLStripCharFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
@Override
public LegacyHTMLStripCharFilter create(Reader input) {
return new LegacyHTMLStripCharFilter(input);
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java Thu May 30 07:53:18 2013
@@ -71,14 +71,17 @@ public class ReversedWildcardFilterFacto
private int minTrailing;
private float maxFractionAsterisk;
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- withOriginal = getBoolean("withOriginal", true);
- maxPosAsterisk = getInt("maxPosAsterisk", 2);
- maxPosQuestion = getInt("maxPosQuestion", 1);
- minTrailing = getInt("minTrailing", 2);
- maxFractionAsterisk = getFloat("maxFractionAsterisk", 0.0f);
+ /** Creates a new ReversedWildcardFilterFactory */
+ public ReversedWildcardFilterFactory(Map<String,String> args) {
+ super(args);
+ withOriginal = getBoolean(args, "withOriginal", true);
+ maxPosAsterisk = getInt(args, "maxPosAsterisk", 2);
+ maxPosQuestion = getInt(args, "maxPosQuestion", 1);
+ minTrailing = getInt(args, "minTrailing", 2);
+ maxFractionAsterisk = getFloat(args, "maxFractionAsterisk", 0.0f);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
@@ -133,13 +136,4 @@ public class ReversedWildcardFilterFacto
public char getMarkerChar() {
return markerChar;
}
-
- protected float getFloat(String name, float defValue) {
- String val = args.get(name);
- if (val == null) {
- return defValue;
- } else {
- return Float.parseFloat(val);
- }
- }
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Thu May 30 07:53:18 2013
@@ -17,15 +17,22 @@
package org.apache.solr.analysis;
import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.DateField;
import static org.apache.solr.schema.TrieField.TrieTypes;
import java.io.IOException;
import java.io.Reader;
+import java.util.HashMap;
+import java.util.Iterator;
/**
* Tokenizer for trie fields. It uses NumericTokenStream to create multiple trie encoded string per number.
@@ -44,13 +51,14 @@ public class TrieTokenizerFactory extend
protected final TrieTypes type;
public TrieTokenizerFactory(TrieTypes type, int precisionStep) {
+ super(new HashMap<String,String>());
this.type = type;
this.precisionStep = precisionStep;
}
@Override
- public TrieTokenizer create(Reader input) {
- return new TrieTokenizer(input, type, TrieTokenizer.getNumericTokenStream(precisionStep));
+ public TrieTokenizer create(AttributeFactory factory, Reader input) {
+ return new TrieTokenizer(input, type, TrieTokenizer.getNumericTokenStream(factory, precisionStep));
}
}
@@ -59,36 +67,47 @@ final class TrieTokenizer extends Tokeni
protected final TrieTypes type;
protected final NumericTokenStream ts;
+ // NumericTokenStream does not support CharTermAttribute so keep it local
+ private final CharTermAttribute termAtt = new CharTermAttributeImpl();
protected final OffsetAttribute ofsAtt = addAttribute(OffsetAttribute.class);
protected int startOfs, endOfs;
protected boolean hasValue;
- protected final char[] buf = new char[32];
- static NumericTokenStream getNumericTokenStream(int precisionStep) {
- return new NumericTokenStream(precisionStep);
+ static NumericTokenStream getNumericTokenStream(AttributeFactory factory, int precisionStep) {
+ return new NumericTokenStream(factory, precisionStep);
}
- public TrieTokenizer(Reader input, TrieTypes type, NumericTokenStream ts) {
- // must share the attribute source with the NumericTokenStream we delegate to
- super(ts, input);
+ public TrieTokenizer(Reader input, TrieTypes type, final NumericTokenStream ts) {
+ // Häckidy-Hick-Hack: must share the attributes with the NumericTokenStream we delegate to, so we create a fake factory:
+ super(new AttributeFactory() {
+ @Override
+ public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+ return (AttributeImpl) ts.addAttribute(attClass);
+ }
+ }, input);
+ // add all attributes:
+ for (Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator(); it.hasNext();) {
+ addAttribute(it.next());
+ }
this.type = type;
this.ts = ts;
+ // dates tend to be longer, especially when math is involved
+ termAtt.resizeBuffer( type == TrieTypes.DATE ? 128 : 32 );
}
@Override
public void reset() {
try {
int upto = 0;
- while (upto < buf.length) {
- final int length = input.read(buf, upto, buf.length - upto);
+ char[] buf = termAtt.buffer();
+ while (true) {
+ final int length = input.read(buf, upto, buf.length-upto);
if (length == -1) break;
upto += length;
+ if (upto == buf.length)
+ buf = termAtt.resizeBuffer(1+buf.length);
}
- // skip remaining data if buffer was too short:
- if (upto == buf.length) {
- input.skip(Long.MAX_VALUE);
- }
-
+ termAtt.setLength(upto);
this.startOfs = correctOffset(0);
this.endOfs = correctOffset(upto);