You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/05/07 13:21:14 UTC
svn commit: r1479862 [26/38] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/solr/ dev-tools/maven/solr/core/src/java/
dev-tools/maven/solr/solrj/src/java/ dev-t...
Modified: lucene/dev/branches/lucene4258/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java Tue May 7 11:20:55 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.util.T
import org.apache.lucene.analysis.commongrams.CommonGramsFilterFactory;
import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
-import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.core.SolrCore;
import org.carrot2.core.LanguageCode;
import org.carrot2.core.attribute.Init;
import org.carrot2.core.attribute.Processing;
@@ -59,8 +59,8 @@ public class SolrStopwordsCarrot2Lexical
@Init
@Input
- @Attribute(key = "solrIndexSchema")
- private IndexSchema schema;
+ @Attribute(key = "solrCore")
+ private SolrCore core;
@Processing
@Input
@@ -86,7 +86,7 @@ public class SolrStopwordsCarrot2Lexical
// No need to synchronize here, Carrot2 ensures that instances
// of this class are not used by multiple threads at a time.
if (!solrStopWords.containsKey(fieldName)) {
- final Analyzer fieldAnalyzer = schema.getFieldType(fieldName)
+ final Analyzer fieldAnalyzer = core.getLatestSchema().getFieldType(fieldName)
.getAnalyzer();
if (fieldAnalyzer instanceof TokenizerChain) {
final TokenFilterFactory[] filterFactories = ((TokenizerChain) fieldAnalyzer)
Modified: lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml Tue May 7 11:20:55 2013
@@ -165,13 +165,10 @@
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<!-- Case insensitive stop word removal.
- enablePositionIncrements=true ensures that a 'gap' is left to
- allow for accurate phrase queries.
-->
<filter class="solr.StopFilterFactory"
ignoreCase="true"
words="stopwords.txt"
- enablePositionIncrements="true"
/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
@@ -265,9 +262,6 @@
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
- compressed: [false] if this field should be stored using gzip compression
- (this will only apply if the field type is compressable; among
- the standard field types, only TextField and StrField are)
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java Tue May 7 11:20:55 2013
@@ -22,6 +22,8 @@ import org.apache.tika.metadata.Metadata
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.html.HtmlMapper;
+import org.apache.tika.parser.html.IdentityHtmlMapper;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerDecorator;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -63,7 +65,7 @@ public class TikaEntityProcessor extends
private boolean done = false;
private String parser;
static final String AUTO_PARSER = "org.apache.tika.parser.AutoDetectParser";
-
+ private String htmlMapper;
@Override
protected void firstInit(Context context) {
@@ -88,6 +90,13 @@ public class TikaEntityProcessor extends
format = "text";
if (!"html".equals(format) && !"xml".equals(format) && !"text".equals(format)&& !"none".equals(format) )
throw new DataImportHandlerException(SEVERE, "'format' can be one of text|html|xml|none");
+
+ htmlMapper = context.getResolvedEntityAttribute("htmlMapper");
+ if (htmlMapper == null)
+ htmlMapper = "default";
+ if (!"default".equals(htmlMapper) && !"identity".equals(htmlMapper))
+ throw new DataImportHandlerException(SEVERE, "'htmlMapper', if present, must be 'default' or 'identity'");
+
parser = context.getResolvedEntityAttribute("parser");
if(parser == null) {
parser = AUTO_PARSER;
@@ -124,7 +133,11 @@ public class TikaEntityProcessor extends
tikaParser = context.getSolrCore().getResourceLoader().newInstance(parser, Parser.class);
}
try {
- tikaParser.parse(is, contentHandler, metadata , new ParseContext());
+ ParseContext context = new ParseContext();
+ if ("identity".equals(htmlMapper)){
+ context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
+ }
+ tikaParser.parse(is, contentHandler, metadata , context);
} catch (Exception e) {
wrapAndThrow(SEVERE, e, "Unable to read content");
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java Tue May 7 11:20:55 2013
@@ -65,7 +65,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top1 did not return 2 messages", swi.docs.size(), 2);
@@ -80,7 +80,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its children did not return 8 messages", swi.docs.size(), 8);
@@ -96,7 +96,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 5 messages", swi.docs.size(), 5);
@@ -112,7 +112,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
@@ -129,7 +129,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
@@ -145,7 +145,7 @@ public class TestMailEntityProcessor ext
DataImporter di = new DataImporter();
di.loadAndInit(getConfigFromMap(paramMap));
Entity ent = di.getConfig().getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java Tue May 7 11:20:55 2013
@@ -55,6 +55,21 @@ public class TestTikaEntityProcessor ext
,"//str[@name='text']"
};
+ private String[] testsHTMLDefault = {
+ "//*[@numFound='1']"
+ , "//str[@name='text'][contains(.,'Basic div')]"
+ , "//str[@name='text'][contains(.,'<h1>')]"
+ , "//str[@name='text'][not(contains(.,'<div>'))]" //default mapper lower-cases elements as it maps
+ , "//str[@name='text'][not(contains(.,'<DIV>'))]"
+ };
+
+ private String[] testsHTMLIdentity = {
+ "//*[@numFound='1']"
+ , "//str[@name='text'][contains(.,'Basic div')]"
+ , "//str[@name='text'][contains(.,'<h1>')]"
+ , "//str[@name='text'][contains(.,'<div>')]"
+ , "//str[@name='text'][contains(.,'class=\"classAttribute\"')]" //attributes are lower-cased
+ };
@BeforeClass
public static void beforeClass() throws Exception {
@@ -67,4 +82,36 @@ public class TestTikaEntityProcessor ext
assertQ(req("*:*"), tests );
}
+ @Test
+ public void testTikaHTMLMapperEmpty() throws Exception {
+ runFullImport(getConfigHTML(null));
+ assertQ(req("*:*"), testsHTMLDefault);
+ }
+
+ @Test
+ public void testTikaHTMLMapperDefault() throws Exception {
+ runFullImport(getConfigHTML("default"));
+ assertQ(req("*:*"), testsHTMLDefault);
+ }
+
+ @Test
+ public void testTikaHTMLMapperIdentity() throws Exception {
+ runFullImport(getConfigHTML("identity"));
+ assertQ(req("*:*"), testsHTMLIdentity);
+ }
+
+ private String getConfigHTML(String htmlMapper) {
+ return
+ "<dataConfig>" +
+ " <dataSource type='BinFileDataSource'/>" +
+ " <document>" +
+ " <entity name='Tika' format='xml' processor='TikaEntityProcessor' " +
+ " url='" + getFile("dihextras/structured.html").getAbsolutePath() + "' " +
+ ((htmlMapper == null) ? "" : (" htmlMapper='" + htmlMapper + "'")) + ">" +
+ " <field column='text'/>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>";
+
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Tue May 7 11:20:55 2013
@@ -124,7 +124,7 @@ public class DataImportHandler extends R
}
SolrParams params = req.getParams();
NamedList defaultParams = (NamedList) initArgs.get("defaults");
- RequestInfo requestParams = new RequestInfo(getParamsMap(params), contentStream);
+ RequestInfo requestParams = new RequestInfo(req, getParamsMap(params), contentStream);
String command = requestParams.getCommand();
if (DataImporter.SHOW_CONF_CMD.equals(command)) {
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Tue May 7 11:20:55 2013
@@ -20,9 +20,7 @@ package org.apache.solr.handler.dataimpo
import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.SystemIdResolver;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.XMLErrorLogger;
import org.apache.solr.handler.dataimport.config.ConfigNameConstants;
@@ -36,7 +34,6 @@ import static org.apache.solr.handler.da
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
import static org.apache.solr.handler.dataimport.DocBuilder.loadClass;
import static org.apache.solr.handler.dataimport.config.ConfigNameConstants.CLASS;
-import static org.apache.solr.handler.dataimport.config.ConfigNameConstants.NAME;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,8 +48,14 @@ import javax.xml.parsers.DocumentBuilder
import java.io.IOException;
import java.io.StringReader;
-import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@@ -86,7 +89,6 @@ public class DataImporter {
private ReentrantLock importLock = new ReentrantLock();
private boolean isDeltaImportSupported = false;
private final String handlerName;
- private Map<String, SchemaField> lowerNameVsSchemaField = new HashMap<String, SchemaField>();
/**
* Only for testing purposes
@@ -98,8 +100,7 @@ public class DataImporter {
DataImporter(SolrCore core, String handlerName) {
this.handlerName = handlerName;
this.core = core;
- this.schema = core.getSchema();
- loadSchemaFieldMap();
+ this.schema = core.getLatestSchema();
}
@@ -110,8 +111,13 @@ public class DataImporter {
if (importLock.tryLock()) {
boolean success = false;
try {
+ if (null != params.getRequest()) {
+ if (schema != params.getRequest().getSchema()) {
+ schema = params.getRequest().getSchema();
+ }
+ }
String dataConfigText = params.getDataConfig();
- String dataconfigFile = (String) params.getConfigFile();
+ String dataconfigFile = params.getConfigFile();
InputSource is = null;
if(dataConfigText!=null && dataConfigText.length()>0) {
is = new InputSource(new StringReader(dataConfigText));
@@ -161,31 +167,14 @@ public class DataImporter {
- private void loadSchemaFieldMap() {
- Map<String, SchemaField> modLnvsf = new HashMap<String, SchemaField>();
- for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
- modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
- }
- lowerNameVsSchemaField = Collections.unmodifiableMap(modLnvsf);
- }
-
- public SchemaField getSchemaField(String caseInsensitiveName) {
- SchemaField schemaField = null;
- if(schema!=null) {
- schemaField = schema.getFieldOrNull(caseInsensitiveName);
- }
- if (schemaField == null) {
- schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
- }
- return schemaField;
+ public String getHandlerName() {
+ return handlerName;
}
- public String getHandlerName() {
- return handlerName;
- }
-
-
-
+ public IndexSchema getSchema() {
+ return schema;
+ }
+
/**
* Used by tests
*/
@@ -581,11 +570,7 @@ public class DataImporter {
public static final String TOTAL_DOCS_SKIPPED = "Total Documents Skipped";
}
- public IndexSchema getSchema() {
- return schema;
- }
-
- SolrCore getCore() {
+ public SolrCore getCore() {
return core;
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue May 7 11:20:55 2013
@@ -26,7 +26,9 @@ import org.apache.solr.handler.dataimpor
import org.apache.solr.handler.dataimport.config.EntityField;
import static org.apache.solr.handler.dataimport.SolrWriter.LAST_INDEX_KEY;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -625,11 +627,12 @@ public class DocBuilder {
if (value == null) continue;
if (key.startsWith("$")) continue;
Set<EntityField> field = entity.getColNameVsField().get(key);
- if (field == null && dataImporter.getSchema() != null) {
+ IndexSchema schema = null == reqParams.getRequest() ? null : reqParams.getRequest().getSchema();
+ if (field == null && schema != null) {
// This can be a dynamic field or a field which does not have an entry in data-config ( an implicit field)
- SchemaField sf = dataImporter.getSchema().getFieldOrNull(key);
+ SchemaField sf = schema.getFieldOrNull(key);
if (sf == null) {
- sf = dataImporter.getSchemaField(key);
+ sf = config.getSchemaField(key);
}
if (sf != null) {
addFieldToDoc(entry.getValue(), sf.getName(), 1.0f, sf.multiValued(), doc);
@@ -643,7 +646,7 @@ public class DocBuilder {
boolean toWrite = f.isToWrite();
if(f.isDynamicName()){
name = vr.replaceTokens(name);
- SchemaField schemaField = dataImporter.getSchemaField(name);
+ SchemaField schemaField = config.getSchemaField(name);
if(schemaField == null) {
toWrite = false;
} else {
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/RequestInfo.java Tue May 7 11:20:55 2013
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrQueryRequest;
public class RequestInfo {
private final String command;
@@ -37,13 +38,15 @@ public class RequestInfo {
private final List<String> entitiesToRun;
private final Map<String,Object> rawParams;
private final String configFile;
- private final String dataConfig;
+ private final String dataConfig;
+ private final SolrQueryRequest request;
//TODO: find a different home for these two...
private final ContentStream contentStream;
private final DebugInfo debugInfo;
- public RequestInfo(Map<String,Object> requestParams, ContentStream stream) {
+ public RequestInfo(SolrQueryRequest request, Map<String,Object> requestParams, ContentStream stream) {
+ this.request = request;
this.contentStream = stream;
if (requestParams.containsKey("command")) {
command = (String) requestParams.get("command");
@@ -167,4 +170,8 @@ public class RequestInfo {
public String getConfigFile() {
return configFile;
}
+
+ public SolrQueryRequest getRequest() {
+ return request;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigParseUtil.java Tue May 7 11:20:55 2013
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +33,6 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class ConfigParseUtil {
- private static final Logger LOG = LoggerFactory .getLogger(ConfigParseUtil.class);
public static String getStringAttribute(Element e, String name, String def) {
String r = e.getAttribute(name);
@@ -77,46 +77,4 @@ public class ConfigParseUtil {
}
return result;
}
-
- public static void verifyWithSchema(DataImporter di, Map<String,EntityField> fields) {
- Map<String,SchemaField> schemaFields = null;
- if (di.getSchema() == null) {
- schemaFields = Collections.emptyMap();
- } else {
- schemaFields = di.getSchema().getFields();
- }
- for (Map.Entry<String,SchemaField> entry : schemaFields.entrySet()) {
- SchemaField sf = entry.getValue();
- if (!fields.containsKey(sf.getName())) {
- if (sf.isRequired()) {
- LOG
- .info(sf.getName()
- + " is a required field in SolrSchema . But not found in DataConfig");
- }
- }
- }
- for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
- EntityField fld = entry.getValue();
- SchemaField field = di.getSchemaField(fld.getName());
- if (field == null) {
- LOG
- .info("The field :"
- + fld.getName()
- + " present in DataConfig does not have a counterpart in Solr Schema");
- }
- }
- }
-
- public static Map<String,EntityField> gatherAllFields(DataImporter di, Entity e) {
- Map<String,EntityField> fields = new HashMap<String,EntityField>();
- if (e.getFields() != null) {
- for (EntityField f : e.getFields()) {
- fields.put(f.getName(), f);
- }
- }
- for (Entity e1 : e.getChildren()) {
- fields.putAll(gatherAllFields(di, e1));
- }
- return fields;
- }
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java Tue May 7 11:20:55 2013
@@ -2,10 +2,16 @@ package org.apache.solr.handler.dataimpo
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
/*
@@ -41,6 +47,8 @@ import org.w3c.dom.Element;
* @since solr 1.3
*/
public class DIHConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(DIHConfiguration.class);
+
// TODO - remove from here and add it to entity
private final String deleteQuery;
@@ -51,10 +59,14 @@ public class DIHConfiguration {
private final Script script;
private final Map<String, Map<String,String>> dataSources;
private final PropertyWriter propertyWriter;
+ private final IndexSchema schema;
+ private final Map<String,SchemaField> lowerNameVsSchemaField;
public DIHConfiguration(Element element, DataImporter di,
List<Map<String,String>> functions, Script script,
Map<String,Map<String,String>> dataSources, PropertyWriter pw) {
+ schema = di.getSchema();
+ lowerNameVsSchemaField = null == schema ? Collections.<String,SchemaField>emptyMap() : loadSchemaFieldMap();
this.deleteQuery = ConfigParseUtil.getStringAttribute(element, "deleteQuery", null);
this.onImportStart = ConfigParseUtil.getStringAttribute(element, "onImportStart", null);
this.onImportEnd = ConfigParseUtil.getStringAttribute(element, "onImportEnd", null);
@@ -62,9 +74,9 @@ public class DIHConfiguration {
List<Element> l = ConfigParseUtil.getChildNodes(element, "entity");
boolean docRootFound = false;
for (Element e : l) {
- Entity entity = new Entity(docRootFound, e, di, null);
- Map<String, EntityField> fields = ConfigParseUtil.gatherAllFields(di, entity);
- ConfigParseUtil.verifyWithSchema(di, fields);
+ Entity entity = new Entity(docRootFound, e, di, this, null);
+ Map<String, EntityField> fields = gatherAllFields(di, entity);
+ verifyWithSchema(fields);
modEntities.add(entity);
}
this.entities = Collections.unmodifiableList(modEntities);
@@ -80,6 +92,64 @@ public class DIHConfiguration {
this.dataSources = Collections.unmodifiableMap(dataSources);
this.propertyWriter = pw;
}
+
+ private void verifyWithSchema(Map<String,EntityField> fields) {
+ Map<String,SchemaField> schemaFields = null;
+ if (schema == null) {
+ schemaFields = Collections.emptyMap();
+ } else {
+ schemaFields = schema.getFields();
+ }
+ for (Map.Entry<String,SchemaField> entry : schemaFields.entrySet()) {
+ SchemaField sf = entry.getValue();
+ if (!fields.containsKey(sf.getName())) {
+ if (sf.isRequired()) {
+ LOG.info(sf.getName() + " is a required field in SolrSchema . But not found in DataConfig");
+ }
+ }
+ }
+ for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
+ EntityField fld = entry.getValue();
+ SchemaField field = getSchemaField(fld.getName());
+ if (field == null) {
+ LOG.info("The field :" + fld.getName() + " present in DataConfig does not have a counterpart in Solr Schema");
+ }
+ }
+ }
+
+ private Map<String,EntityField> gatherAllFields(DataImporter di, Entity e) {
+ Map<String,EntityField> fields = new HashMap<String,EntityField>();
+ if (e.getFields() != null) {
+ for (EntityField f : e.getFields()) {
+ fields.put(f.getName(), f);
+ }
+ }
+ for (Entity e1 : e.getChildren()) {
+ fields.putAll(gatherAllFields(di, e1));
+ }
+ return fields;
+ }
+
+ private Map<String,SchemaField> loadSchemaFieldMap() {
+ Map<String, SchemaField> modLnvsf = new HashMap<String, SchemaField>();
+ for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
+ modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
+ }
+ return Collections.unmodifiableMap(modLnvsf);
+ }
+
+ public SchemaField getSchemaField(String caseInsensitiveName) {
+ SchemaField schemaField = null;
+ if(schema!=null) {
+ schemaField = schema.getFieldOrNull(caseInsensitiveName);
+ }
+ if (schemaField == null) {
+ schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
+ }
+ return schemaField;
+ }
+
+
public String getDeleteQuery() {
return deleteQuery;
}
@@ -104,4 +174,8 @@ public class DIHConfiguration {
public PropertyWriter getPropertyWriter() {
return propertyWriter;
}
+
+ public IndexSchema getSchema() {
+ return schema;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/Entity.java Tue May 7 11:20:55 2013
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.solr.handler.dataimport.DataImportHandlerException;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.w3c.dom.Element;
@@ -45,9 +46,11 @@ public class Entity {
private final Map<String,Set<EntityField>> colNameVsField;
private final Map<String,String> allAttributes;
private final List<Map<String,String>> allFieldAttributes;
+ private final DIHConfiguration config;
- public Entity(boolean docRootFound, Element element, DataImporter di, Entity parent) {
+ public Entity(boolean docRootFound, Element element, DataImporter di, DIHConfiguration config, Entity parent) {
this.parentEntity = parent;
+ this.config = config;
String modName = ConfigParseUtil.getStringAttribute(element, ConfigNameConstants.NAME, null);
if (modName == null) {
@@ -85,13 +88,13 @@ public class Entity {
List<Map<String,String>> modAllFieldAttributes = new ArrayList<Map<String,String>>();
for (Element elem : n) {
EntityField.Builder fieldBuilder = new EntityField.Builder(elem);
- if (di.getSchema() != null) {
+ if (config.getSchema() != null) {
if (fieldBuilder.getNameOrColumn() != null
&& fieldBuilder.getNameOrColumn().contains("${")) {
fieldBuilder.dynamicName = true;
} else {
- SchemaField schemaField = di.getSchemaField(fieldBuilder
- .getNameOrColumn());
+ SchemaField schemaField = config.getSchemaField
+ (fieldBuilder.getNameOrColumn());
if (schemaField != null) {
fieldBuilder.name = schemaField.getName();
fieldBuilder.multiValued = schemaField.multiValued();
@@ -139,8 +142,8 @@ public class Entity {
.unmodifiableList(modAllFieldAttributes);
String modPkMappingFromSchema = null;
- if (di.getSchema() != null) {
- SchemaField uniqueKey = di.getSchema().getUniqueKeyField();
+ if (config.getSchema() != null) {
+ SchemaField uniqueKey = config.getSchema().getUniqueKeyField();
if (uniqueKey != null) {
modPkMappingFromSchema = uniqueKey.getName();
// if no fields are mentioned . solr uniqueKey is same as dih 'pk'
@@ -160,8 +163,7 @@ public class Entity {
n = ConfigParseUtil.getChildNodes(element, "entity");
List<Entity> modEntities = new ArrayList<Entity>();
for (Element elem : n) {
- modEntities
- .add(new Entity((docRootFound || this.docRoot), elem, di, this));
+ modEntities.add(new Entity((docRootFound || this.docRoot), elem, di, config, this));
}
this.children = Collections.unmodifiableList(modEntities);
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml Tue May 7 11:20:55 2013
@@ -246,9 +246,6 @@
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
- compressed: [false] if this field should be stored using gzip compression
- (this will only apply if the field type is compressable; among
- the standard field types, only TextField and StrField are)
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContextImpl.java Tue May 7 11:20:55 2013
@@ -37,7 +37,7 @@ public class TestContextImpl extends Abs
public void testCoreScope() {
DataImporter di = new DataImporter();
di.loadAndInit("<dataConfig><document /></dataConfig>");
- DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(new HashMap<String,Object>(), null));
+ DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(null, new HashMap<String,Object>(), null));
ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, db);
String lala = new String("lala");
ctx.setSessionAttribute("huhu", lala, Context.SCOPE_SOLR_CORE);
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java Tue May 7 11:20:55 2013
@@ -57,7 +57,7 @@ public class TestDocBuilder extends Abst
DIHConfiguration cfg = di.getConfig();
Entity ent = cfg.getEntities().get(0);
MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
@@ -77,7 +77,7 @@ public class TestDocBuilder extends Abst
Entity ent = cfg.getEntities().get(0);
MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "delta-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.FALSE, swi.deleteAllCalled);
@@ -98,7 +98,7 @@ public class TestDocBuilder extends Abst
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
@@ -127,7 +127,7 @@ public class TestDocBuilder extends Abst
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(createMap("command", "import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
SolrWriterImpl swi = new SolrWriterImpl();
di.runCmd(rp, swi);
assertEquals(Boolean.FALSE, swi.deleteAllCalled);
@@ -153,7 +153,7 @@ public class TestDocBuilder extends Abst
di.loadAndInit(dc_singleEntity);
DIHConfiguration cfg = di.getConfig();
Entity ent = cfg.getEntities().get(0);
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("id", 1, "desc", "one"));
l.add(createMap("id", 2, "desc", "two"));
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java Tue May 7 11:20:55 2013
@@ -36,7 +36,7 @@ public class TestFieldReader extends Abs
DataImporter di = new DataImporter();
di.loadAndInit(config);
TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("xml", xml));
MockDataSource.setIterator("select * from a", l.iterator());
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java Tue May 7 11:20:55 2013
@@ -34,7 +34,7 @@ public class TestPlainTextEntityProcesso
DataImporter di = new DataImporter();
di.loadAndInit(DATA_CONFIG);
TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(createMap("command", "full-import"), null);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
di.runCmd(rp, sw);
assertEquals(DS.s, sw.docs.get(0).getFieldValue("x"));
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java Tue May 7 11:20:55 2013
@@ -53,9 +53,9 @@ public class TestScriptTransformer exten
EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
sep.init(context);
sep.applyTransformer(map);
- assertEquals(map.get("name"), "Hello Scott");
+ assertEquals("Hello Scott", map.get("name").toString());
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
@@ -86,9 +86,9 @@ public class TestScriptTransformer exten
EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
sep.init(context);
sep.applyTransformer(map);
- assertEquals(map.get("name"), "Hello Scott");
+ assertEquals("Hello Scott", map.get("name").toString());
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
@@ -104,7 +104,7 @@ public class TestScriptTransformer exten
DIHConfiguration dc = di.readFromXml(document);
assertTrue(dc.getScript().getText().indexOf("checkNextToken") > -1);
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
@@ -131,7 +131,7 @@ public class TestScriptTransformer exten
sep.applyTransformer(map);
assertNull(map.get("$hasMore"));
} catch (DataImportHandlerException e) {
- assumeFalse("This JVM does not have Rhino installed. Test Skipped.", e
+ assumeFalse("This JVM does not have JavaScript installed. Test Skipped.", e
.getMessage().startsWith("Cannot load Script Engine for language"));
throw e;
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue May 7 11:20:55 2013
@@ -28,10 +28,10 @@ import org.apache.solr.common.params.Upd
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.loader.ContentStreamLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.IndexSchema;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.tika.config.TikaConfig;
@@ -81,7 +81,7 @@ public class ExtractingDocumentLoader ex
private static final XPathParser PARSER =
new XPathParser("xhtml", XHTMLContentHandler.XHTML);
- final IndexSchema schema;
+ final SolrCore core;
final SolrParams params;
final UpdateRequestProcessor processor;
final boolean ignoreTikaException;
@@ -95,7 +95,7 @@ public class ExtractingDocumentLoader ex
public ExtractingDocumentLoader(SolrQueryRequest req, UpdateRequestProcessor processor,
TikaConfig config, SolrContentHandlerFactory factory) {
this.params = req.getParams();
- schema = req.getSchema();
+ this.core = req.getCore();
this.config = config;
this.processor = processor;
@@ -167,7 +167,7 @@ public class ExtractingDocumentLoader ex
String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);
boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false);
- SolrContentHandler handler = factory.createSolrContentHandler(metadata, params, schema);
+ SolrContentHandler handler = factory.createSolrContentHandler(metadata, params, req.getSchema());
ContentHandler parsingHandler = handler;
StringWriter writer = null;
Modified: lucene/dev/branches/lucene4258/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml Tue May 7 11:20:55 2013
@@ -285,7 +285,7 @@
<fieldtype name="syn" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory"/>
- <filter name="syn" class="solr.SynonymFilterFactory" synonyms="synonyms.txt"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"/>
</analyzer>
</fieldtype>
@@ -354,8 +354,8 @@
termPositions="true" termOffsets="true"/>
<!-- test highlit field settings -->
- <field name="test_hlt" type="highlittext" indexed="true" compressed="true"/>
- <field name="test_hlt_off" type="highlittext" indexed="true" compressed="false"/>
+ <field name="test_hlt" type="highlittext" indexed="true"/>
+ <field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
<field name="teststop" type="teststop" indexed="true" stored="true"/>
Modified: lucene/dev/branches/lucene4258/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java Tue May 7 11:20:55 2013
@@ -92,7 +92,7 @@ public class UIMAUpdateRequestProcessor
} catch (Exception e) {
String logField = solrUIMAConfiguration.getLogField();
if(logField == null){
- SchemaField uniqueKeyField = solrCore.getSchema().getUniqueKeyField();
+ SchemaField uniqueKeyField = cmd.getReq().getSchema().getUniqueKeyField();
if(uniqueKeyField != null){
logField = uniqueKeyField.getName();
}
Modified: lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml Tue May 7 11:20:55 2013
@@ -253,11 +253,9 @@
-->
<!--
Case insensitive stop word removal. add
- enablePositionIncrements=true in both the index and query
- analyzers to leave a 'gap' for more accurate phrase queries.
-->
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
catenateNumbers="1" catenateAll="0" splitOnCaseChange="1" />
@@ -269,7 +267,7 @@
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"
ignoreCase="true" expand="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="0"
catenateNumbers="0" catenateAll="0" splitOnCaseChange="1" />
@@ -316,7 +314,7 @@
<analyzer type="index">
<tokenizer class="solr.MockTokenizerFactory" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
catenateNumbers="1" catenateAll="0" splitOnCaseChange="0" />
@@ -327,7 +325,7 @@
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"
ignoreCase="true" expand="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="0"
catenateNumbers="0" catenateAll="0" splitOnCaseChange="0" />
@@ -346,7 +344,7 @@
<analyzer type="index">
<tokenizer class="solr.MockTokenizerFactory" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
catenateNumbers="1" catenateAll="0" splitOnCaseChange="0" />
@@ -360,7 +358,7 @@
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt"
ignoreCase="true" expand="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true"
- words="stopwords.txt" enablePositionIncrements="true" />
+ words="stopwords.txt" />
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="0"
catenateNumbers="0" catenateAll="0" splitOnCaseChange="0" />
@@ -470,10 +468,7 @@
field type: mandatory - the name of a previously defined type from
the <types> section indexed: true if this field should be indexed
(searchable or sortable) stored: true if this field should be
- retrievable compressed: [false] if this field should be stored
- using gzip compression (this will only apply if the field type is
- compressable; among the standard field types, only TextField and
- StrField are) multiValued: true if this field may contain multiple
+ retrievable multiValued: true if this field may contain multiple
values per document omitNorms: (expert) set to true to omit the
norms associated with this field (this disables length
normalization and index-time boosting for the field, and saves
Modified: lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml Tue May 7 11:20:55 2013
@@ -253,8 +253,6 @@
-->
<!--
Case insensitive stop word removal. add
- enablePositionIncrements=true in both the index and query
- analyzers to leave a 'gap' for more accurate phrase queries.
-->
<filter class="solr.WordDelimiterFilterFactory"
generateWordParts="1" generateNumberParts="1" catenateWords="1"
@@ -466,10 +464,7 @@
field type: mandatory - the name of a previously defined type from
the <types> section indexed: true if this field should be indexed
(searchable or sortable) stored: true if this field should be
- retrievable compressed: [false] if this field should be stored
- using gzip compression (this will only apply if the field type is
- compressable; among the standard field types, only TextField and
- StrField are) multiValued: true if this field may contain multiple
+ retrievable multiValued: true if this field may contain multiple
values per document omitNorms: (expert) set to true to omit the
norms associated with this field (this disables length
normalization and index-time boosting for the field, and saves
Modified: lucene/dev/branches/lucene4258/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/uima/src/test/org/apache/solr/uima/analysis/UIMATokenizersSolrIntegrationTest.java Tue May 7 11:20:55 2013
@@ -19,6 +19,7 @@ package org.apache.solr.uima.analysis;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.IndexSchema;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,10 +35,11 @@ public class UIMATokenizersSolrIntegrati
@Test
public void testInitialization() throws Exception {
- assertNotNull(h.getCore().getSchema().getField("sentences"));
- assertNotNull(h.getCore().getSchema().getFieldType("sentences"));
- assertNotNull(h.getCore().getSchema().getField("nouns"));
- assertNotNull(h.getCore().getSchema().getFieldType("nouns"));
+ IndexSchema schema = h.getCore().getLatestSchema();
+ assertNotNull(schema.getField("sentences"));
+ assertNotNull(schema.getFieldType("sentences"));
+ assertNotNull(schema.getField("nouns"));
+ assertNotNull(schema.getFieldType("nouns"));
}
@Test
Modified: lucene/dev/branches/lucene4258/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/ivy.xml?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/ivy.xml (original)
+++ lucene/dev/branches/lucene4258/solr/core/ivy.xml Tue May 7 11:20:55 2013
@@ -24,7 +24,7 @@
<dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
<dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
<dependency org="commons-lang" name="commons-lang" rev="2.6" transitive="false"/>
- <dependency org="com.google.guava" name="guava" rev="13.0.1" transitive="false"/>
+ <dependency org="com.google.guava" name="guava" rev="14.0.1" transitive="false"/>
<dependency org="org.easymock" name="easymock" rev="3.0" transitive="false"/>
<dependency org="cglib" name="cglib-nodep" rev="2.2" transitive="false"/>
<dependency org="org.objenesis" name="objenesis" rev="1.2" transitive="false"/>
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java Tue May 7 11:20:55 2013
@@ -19,6 +19,7 @@ package org.apache.solr.analysis;
*/
import java.io.Reader;
+import java.util.Map;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -54,6 +55,14 @@ import org.apache.lucene.analysis.util.C
@Deprecated
public class LegacyHTMLStripCharFilterFactory extends CharFilterFactory {
+ /** Creates a new LegacyHTMLStripCharFilterFactory */
+ public LegacyHTMLStripCharFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
@Override
public LegacyHTMLStripCharFilter create(Reader input) {
return new LegacyHTMLStripCharFilter(input);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java Tue May 7 11:20:55 2013
@@ -71,14 +71,17 @@ public class ReversedWildcardFilterFacto
private int minTrailing;
private float maxFractionAsterisk;
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- withOriginal = getBoolean("withOriginal", true);
- maxPosAsterisk = getInt("maxPosAsterisk", 2);
- maxPosQuestion = getInt("maxPosQuestion", 1);
- minTrailing = getInt("minTrailing", 2);
- maxFractionAsterisk = getFloat("maxFractionAsterisk", 0.0f);
+ /** Creates a new ReversedWildcardFilterFactory */
+ public ReversedWildcardFilterFactory(Map<String,String> args) {
+ super(args);
+ withOriginal = getBoolean(args, "withOriginal", true);
+ maxPosAsterisk = getInt(args, "maxPosAsterisk", 2);
+ maxPosQuestion = getInt(args, "maxPosQuestion", 1);
+ minTrailing = getInt(args, "minTrailing", 2);
+ maxFractionAsterisk = getFloat(args, "maxFractionAsterisk", 0.0f);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
@@ -133,13 +136,4 @@ public class ReversedWildcardFilterFacto
public char getMarkerChar() {
return markerChar;
}
-
- protected float getFloat(String name, float defValue) {
- String val = args.get(name);
- if (val == null) {
- return defValue;
- } else {
- return Float.parseFloat(val);
- }
- }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Tue May 7 11:20:55 2013
@@ -31,6 +31,7 @@ import static org.apache.solr.schema.Tri
import java.io.IOException;
import java.io.Reader;
+import java.util.HashMap;
import java.util.Iterator;
/**
@@ -50,6 +51,7 @@ public class TrieTokenizerFactory extend
protected final TrieTypes type;
public TrieTokenizerFactory(TrieTypes type, int precisionStep) {
+ super(new HashMap<String,String>());
this.type = type;
this.precisionStep = precisionStep;
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/AssignShard.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/AssignShard.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/AssignShard.java Tue May 7 11:20:55 2013
@@ -39,7 +39,7 @@ public class AssignShard {
numShards = 1;
}
String returnShardId = null;
- Map<String, Slice> sliceMap = state.getSlicesMap(collection);
+ Map<String, Slice> sliceMap = state.getActiveSlicesMap(collection);
// TODO: now that we create shards ahead of time, is this code needed? Esp since hash ranges aren't assigned when creating via this method?
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java Tue May 7 11:20:55 2013
@@ -18,6 +18,7 @@ package org.apache.solr.cloud;
*/
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.SolrParams;
public class CloudDescriptor {
@@ -28,6 +29,11 @@ public class CloudDescriptor {
private Integer numShards;
private String nodeName = null;
+ /* shardRange and shardState are used once-only during sub shard creation for shard splits
+ * Use the values from {@link Slice} instead */
+ volatile String shardRange = null;
+ volatile String shardState = Slice.ACTIVE;
+
volatile boolean isLeader = false;
volatile String lastPublished = ZkStateReader.ACTIVE;
@@ -89,4 +95,19 @@ public class CloudDescriptor {
this.nodeName = nodeName;
}
+ public String getShardRange() {
+ return shardRange;
+ }
+
+ public void setShardRange(String shardRange) {
+ this.shardRange = shardRange;
+ }
+
+ public String getShardState() {
+ return shardState;
+ }
+
+ public void setShardState(String shardState) {
+ this.shardState = shardState;
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Tue May 7 11:20:55 2013
@@ -268,6 +268,10 @@ final class ShardLeaderElectionContext e
.getClusterState();
Map<String,Slice> slices = clusterState.getSlicesMap(collection);
Slice slice = slices.get(shardId);
+ if (!slice.getState().equals(Slice.ACTIVE)) {
+ //Return false if the Slice is not active yet.
+ return false;
+ }
Map<String,Replica> replicasMap = slice.getReplicasMap();
for (Map.Entry<String,Replica> shard : replicasMap.entrySet()) {
String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Tue May 7 11:20:55 2013
@@ -142,7 +142,7 @@ public class LeaderElector {
} catch (KeeperException.SessionExpiredException e) {
throw e;
} catch (KeeperException e) {
- SolrException.log(log, "Failed setting watch", e);
+ log.warn("Failed setting watch", e);
// we couldn't set our watch - the node before us may already be down?
// we need to check if we are the leader again
checkIfIamLeader(seq, context, true);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java Tue May 7 11:20:55 2013
@@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -49,7 +50,8 @@ import org.slf4j.LoggerFactory;
*/
public class Overseer {
public static final String QUEUE_OPERATION = "operation";
-
+ public static final String REMOVECOLLECTION = "removecollection";
+
private static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
private static Logger log = LoggerFactory.getLogger(Overseer.class);
@@ -176,6 +178,8 @@ public class Overseer {
clusterState = updateState(clusterState, message);
} else if (DELETECORE.equals(operation)) {
clusterState = removeCore(clusterState, message);
+ } else if (REMOVECOLLECTION.equals(operation)) {
+ clusterState = removeCollection(clusterState, message);
} else if (ZkStateReader.LEADER_PROP.equals(operation)) {
StringBuilder sb = new StringBuilder();
@@ -190,6 +194,10 @@ public class Overseer {
message.getStr(ZkStateReader.SHARD_ID_PROP),
sb.length() > 0 ? sb.toString() : null);
+ } else if ("createshard".equals(operation)) {
+ clusterState = createShard(clusterState, message);
+ } else if ("updateshardstate".equals(operation)) {
+ clusterState = updateShardState(clusterState, message);
} else {
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
@@ -197,6 +205,46 @@ public class Overseer {
return clusterState;
}
+ private ClusterState updateShardState(ClusterState clusterState, ZkNodeProps message) {
+ String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
+ log.info("Update shard state invoked for collection: " + collection);
+ for (String key : message.keySet()) {
+ if (ZkStateReader.COLLECTION_PROP.equals(key)) continue;
+ if (QUEUE_OPERATION.equals(key)) continue;
+
+ Slice slice = clusterState.getSlice(collection, key);
+ if (slice == null) {
+ throw new RuntimeException("Overseer.updateShardState unknown collection: " + collection + " slice: " + key);
+ }
+ log.info("Update shard state " + key + " to " + message.getStr(key));
+ Map<String, Object> props = slice.shallowCopy();
+ props.put(Slice.STATE, message.getStr(key));
+ Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props);
+ clusterState = updateSlice(clusterState, collection, newSlice);
+ }
+
+ return clusterState;
+ }
+
+ private ClusterState createShard(ClusterState clusterState, ZkNodeProps message) {
+ String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
+ String shardId = message.getStr(ZkStateReader.SHARD_ID_PROP);
+ Slice slice = clusterState.getSlice(collection, shardId);
+ if (slice == null) {
+ Map<String, Replica> replicas = Collections.EMPTY_MAP;
+ Map<String, Object> sliceProps = new HashMap<String, Object>();
+ String shardRange = message.getStr(ZkStateReader.SHARD_RANGE_PROP);
+ String shardState = message.getStr(ZkStateReader.SHARD_STATE_PROP);
+ sliceProps.put(Slice.RANGE, shardRange);
+ sliceProps.put(Slice.STATE, shardState);
+ slice = new Slice(shardId, replicas, sliceProps);
+ clusterState = updateSlice(clusterState, collection, slice);
+ } else {
+ log.error("Unable to create Shard: " + shardId + " because it already exists in collection: " + collection);
+ }
+ return clusterState;
+ }
+
private boolean amILeader() {
try {
ZkNodeProps props = ZkNodeProps.load(zkClient.getData("/overseer_elect/leader", null, null, true));
@@ -211,6 +259,7 @@ public class Overseer {
log.info("According to ZK I (id=" + myId + ") am no longer a leader.");
return false;
}
+
/**
* Try to assign core to the cluster.
*/
@@ -247,15 +296,24 @@ public class Overseer {
log.info("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards);
}
sliceName = AssignShard.assignShard(collection, state, numShards);
- log.info("Assigning new node to shard shard=" + sliceName);
+ log.info("Assigning new node to shard=" + sliceName);
}
Slice slice = state.getSlice(collection, sliceName);
+
Map<String,Object> replicaProps = new LinkedHashMap<String,Object>();
replicaProps.putAll(message.getProperties());
// System.out.println("########## UPDATE MESSAGE: " + JSONUtil.toJSON(message));
if (slice != null) {
+ String sliceState = slice.getState();
+
+ // throw an exception if the slice is not yet active.
+
+ //if(!sliceState.equals(Slice.ACTIVE)) {
+ // throw new SolrException(ErrorCode.BAD_REQUEST, "Can not assign core to a non-active slice [" + slice.getName() + "]");
+ //}
+
Replica oldReplica = slice.getReplicasMap().get(coreNodeName);
if (oldReplica != null && oldReplica.containsKey(ZkStateReader.LEADER_PROP)) {
replicaProps.put(ZkStateReader.LEADER_PROP, oldReplica.get(ZkStateReader.LEADER_PROP));
@@ -278,6 +336,9 @@ public class Overseer {
replicaProps.remove(removeKey);
}
replicaProps.remove(ZkStateReader.CORE_NODE_NAME_PROP);
+ // remove shard specific properties
+ String shardRange = (String) replicaProps.remove(ZkStateReader.SHARD_RANGE_PROP);
+ String shardState = (String) replicaProps.remove(ZkStateReader.SHARD_STATE_PROP);
Replica replica = new Replica(coreNodeName, replicaProps);
@@ -292,6 +353,9 @@ public class Overseer {
replicas = slice.getReplicasCopy();
} else {
replicas = new HashMap<String, Replica>(1);
+ sliceProps = new HashMap<String, Object>();
+ sliceProps.put(Slice.RANGE, shardRange);
+ sliceProps.put(Slice.STATE, shardState);
}
replicas.put(replica.getName(), replica);
@@ -399,7 +463,11 @@ public class Overseer {
Slice slice = slices.get(sliceName);
if (slice == null) {
- log.error("Could not mark leader for non existing slice:" + sliceName);
+ slice = coll.getSlice(sliceName);
+ }
+
+ if (slice == null) {
+ log.error("Could not mark leader for non existing/active slice:" + sliceName);
return state;
} else {
// TODO: consider just putting the leader property on the shard, not on individual replicas
@@ -439,6 +507,20 @@ public class Overseer {
}
/*
+ * Remove collection from cloudstate
+ */
+ private ClusterState removeCollection(final ClusterState clusterState, ZkNodeProps message) {
+
+ final String collection = message.getStr("name");
+
+ final Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(clusterState.getCollectionStates()); // shallow copy
+ newCollections.remove(collection);
+
+ ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections);
+ return newState;
+ }
+
+ /*
* Remove core from cloudstate
*/
private ClusterState removeCore(final ClusterState clusterState, ZkNodeProps message) {