You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/05/13 13:18:25 UTC
svn commit: r1102677 [5/6] - in /lucene/dev/branches/docvalues: ./
dev-tools/eclipse/ dev-tools/maven/
dev-tools/maven/solr/contrib/dataimporthandler/src/extras/
dev-tools/maven/solr/src/ dev-tools/maven/solr/src/solrj/
dev-tools/scripts/ lucene/ lucen...
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/README.txt?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/README.txt (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/README.txt Fri May 13 11:18:19 2011
@@ -37,20 +37,26 @@ To start using Solr UIMA Metadata Extrac
</arr>
</lst>
<lst name="fieldMappings">
- <lst name="mapping">
- <str name="type">org.apache.uima.alchemy.ts.concept.ConceptFS</str>
- <str name="feature">text</str>
- <str name="field">concept</str>
+ <lst name="type">
+ <str name="name">org.apache.uima.alchemy.ts.concept.ConceptFS</str>
+ <lst name="mapping">
+ <str name="feature">text</str>
+ <str name="field">concept</str>
+ </lst>
</lst>
- <lst name="mapping">
- <str name="type">org.apache.uima.alchemy.ts.language.LanguageFS</str>
- <str name="feature">language</str>
- <str name="field">language</str>
+ <lst name="type">
+ <str name="name">org.apache.uima.alchemy.ts.language.LanguageFS</str>
+ <lst name="mapping">
+ <str name="feature">language</str>
+ <str name="field">language</str>
+ </lst>
</lst>
- <lst name="mapping">
- <str name="type">org.apache.uima.SentenceAnnotation</str>
- <str name="feature">coveredText</str>
- <str name="field">sentence</str>
+ <lst name="type">
+ <str name="name">org.apache.uima.SentenceAnnotation</str>
+ <lst name="mapping">
+ <str name="feature">coveredText</str>
+ <str name="field">sentence</str>
+ </lst>
</lst>
</lst>
</lst>
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/build.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/build.xml (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/build.xml Fri May 13 11:18:19 2011
@@ -114,6 +114,7 @@
>
<sysproperty key="java.util.logging.config.file" value="${common-solr.dir}/testlogging.properties"/>
<sysproperty key="tests.luceneMatchVersion" value="${tests.luceneMatchVersion}"/>
+ <sysproperty key="tests.codecprovider" value="${tests.codecprovider}"/>
<sysproperty key="tests.codec" value="${tests.codec}"/>
<sysproperty key="tests.locale" value="${tests.locale}"/>
<sysproperty key="tests.timezone" value="${tests.timezone}"/>
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java Fri May 13 11:18:19 2011
@@ -30,14 +30,14 @@ public class SolrUIMAConfiguration {
private boolean fieldsMerging;
- private Map<String, Map<String, String>> typesFeaturesFieldsMapping;
+ private Map<String, Map<String, MapField>> typesFeaturesFieldsMapping;
private String aePath;
private Map<String, Object> runtimeParameters;
public SolrUIMAConfiguration(String aePath, String[] fieldsToAnalyze, boolean fieldsMerging,
- Map<String, Map<String, String>> typesFeaturesFieldsMapping,
+ Map<String, Map<String, MapField>> typesFeaturesFieldsMapping,
Map<String, Object> runtimeParameters) {
this.aePath = aePath;
this.fieldsToAnalyze = fieldsToAnalyze;
@@ -54,7 +54,7 @@ public class SolrUIMAConfiguration {
return fieldsMerging;
}
- public Map<String, Map<String, String>> getTypesFeaturesFieldsMapping() {
+ public Map<String, Map<String, MapField>> getTypesFeaturesFieldsMapping() {
return typesFeaturesFieldsMapping;
}
@@ -65,4 +65,39 @@ public class SolrUIMAConfiguration {
public Map<String, Object> getRuntimeParameters() {
return runtimeParameters;
}
+
+ static final class MapField {
+
+ private String fieldName, fieldNameFeature;
+ private boolean prefix; // valid if dynamicField == true
+ // false: *_s, true: s_*
+
+ MapField(String fieldName, String fieldNameFeature){
+ this.fieldName = fieldName;
+ this.fieldNameFeature = fieldNameFeature;
+ if(fieldNameFeature != null){
+ if(fieldName.startsWith("*")){
+ prefix = false;
+ this.fieldName = fieldName.substring(1);
+ }
+ else if(fieldName.endsWith("*")){
+ prefix = true;
+ this.fieldName = fieldName.substring(0, fieldName.length() - 1);
+ }
+ else
+ throw new RuntimeException("static field name cannot be used for dynamicField");
+ }
+ }
+
+ String getFieldNameFeature(){
+ return fieldNameFeature;
+ }
+
+ String getFieldName(String featureValue){
+ if(fieldNameFeature != null){
+ return prefix ? fieldName + featureValue : featureValue + fieldName;
+ }
+ return fieldName;
+ }
+ }
}
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java Fri May 13 11:18:19 2011
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
/**
* Read configuration for Solr-UIMA integration
@@ -62,18 +63,31 @@ public class SolrUIMAConfigurationReader
}
@SuppressWarnings("rawtypes")
- private Map<String, Map<String, String>> readTypesFeaturesFieldsMapping() {
- Map<String, Map<String, String>> map = new HashMap<String, Map<String, String>>();
+ private Map<String, Map<String, MapField>> readTypesFeaturesFieldsMapping() {
+ Map<String, Map<String, MapField>> map = new HashMap<String, Map<String, MapField>>();
NamedList fieldMappings = (NamedList) args.get("fieldMappings");
/* iterate over UIMA types */
for (int i = 0; i < fieldMappings.size(); i++) {
- NamedList mapping = (NamedList) fieldMappings.get("mapping", i);
- String typeName = (String) mapping.get("type");
- String featureName = (String) mapping.get("feature");
- String mappedFieldName = (String) mapping.get("field");
- Map<String, String> subMap = new HashMap<String, String>();
- subMap.put(featureName, mappedFieldName);
+ NamedList type = (NamedList) fieldMappings.get("type", i);
+ String typeName = (String)type.get("name");
+
+ Map<String, MapField> subMap = new HashMap<String, MapField>();
+ /* iterate over mapping definitions */
+ for(int j = 0; j < type.size() - 1; j++){
+ NamedList mapping = (NamedList) type.get("mapping", j + 1);
+ String featureName = (String) mapping.get("feature");
+ String fieldNameFeature = null;
+ String mappedFieldName = (String) mapping.get("field");
+ if(mappedFieldName == null){
+ fieldNameFeature = (String) mapping.get("fieldNameFeature");
+ mappedFieldName = (String) mapping.get("dynamicField");
+ }
+ if(mappedFieldName == null)
+ throw new RuntimeException("either of field or dynamicField should be defined for feature " + featureName);
+ MapField mapField = new MapField(mappedFieldName, fieldNameFeature);
+ subMap.put(featureName, mapField);
+ }
map.put(typeName, subMap);
}
return map;
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAToSolrMapper.java Fri May 13 11:18:19 2011
@@ -20,6 +20,7 @@ package org.apache.solr.uima.processor;
import java.util.Map;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.FeatureStructure;
import org.apache.uima.cas.Type;
@@ -53,7 +54,7 @@ public class UIMAToSolrMapper {
* name of UIMA type to map
* @param featureFieldsmapping
*/
- public void map(String typeName, Map<String, String> featureFieldsmapping) {
+ public void map(String typeName, Map<String, MapField> featureFieldsmapping) {
try {
FeatureStructure fsMock = (FeatureStructure) Class.forName(typeName).getConstructor(
JCas.class).newInstance(cas);
@@ -62,7 +63,11 @@ public class UIMAToSolrMapper {
.hasNext();) {
FeatureStructure fs = iterator.next();
for (String featureName : featureFieldsmapping.keySet()) {
- String fieldName = featureFieldsmapping.get(featureName);
+ MapField mapField = featureFieldsmapping.get(featureName);
+ String fieldNameFeature = mapField.getFieldNameFeature();
+ String fieldNameFeatureValue = fieldNameFeature == null ? null :
+ fs.getFeatureValueAsString(type.getFeatureByBaseName(fieldNameFeature));
+ String fieldName = mapField.getFieldName(fieldNameFeatureValue);
log.info(new StringBuffer("mapping ").append(typeName).append("@").append(featureName)
.append(" to ").append(fieldName).toString());
String featureValue = null;
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java Fri May 13 11:18:19 2011
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
import org.apache.solr.uima.processor.ae.AEProvider;
import org.apache.solr.uima.processor.ae.AEProviderFactory;
import org.apache.solr.update.AddUpdateCommand;
@@ -39,7 +40,7 @@ import org.apache.uima.resource.Resource
*/
public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor {
- private SolrUIMAConfiguration solrUIMAConfiguration;
+ SolrUIMAConfiguration solrUIMAConfiguration;
private AEProvider aeProvider;
@@ -69,7 +70,7 @@ public class UIMAUpdateRequestProcessor
UIMAToSolrMapper uimaToSolrMapper = new UIMAToSolrMapper(solrInputDocument, jcas);
/* get field mapping from config */
- Map<String, Map<String, String>> typesAndFeaturesFieldsMap = solrUIMAConfiguration
+ Map<String, Map<String, MapField>> typesAndFeaturesFieldsMap = solrUIMAConfiguration
.getTypesFeaturesFieldsMapping();
/* map type features on fields */
for (String typeFQN : typesAndFeaturesFieldsMap.keySet()) {
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Fri May 13 11:18:19 2011
@@ -33,6 +33,8 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.XmlUpdateRequestHandler;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -66,6 +68,26 @@ public class UIMAUpdateRequestProcessorT
UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory) chained
.getFactories()[0];
assertNotNull(factory);
+ UpdateRequestProcessor processor = factory.getInstance(req(), null, null);
+ assertTrue(processor instanceof UIMAUpdateRequestProcessor);
+ }
+
+ @Test
+ public void testMultiMap() {
+ SolrCore core = h.getCore();
+ UpdateRequestProcessorChain chained = core.getUpdateProcessingChain("uima-multi-map");
+ assertNotNull(chained);
+ UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory) chained
+ .getFactories()[0];
+ assertNotNull(factory);
+ UpdateRequestProcessor processor = factory.getInstance(req(), null, null);
+ assertTrue(processor instanceof UIMAUpdateRequestProcessor);
+ SolrUIMAConfiguration conf = ((UIMAUpdateRequestProcessor)processor).solrUIMAConfiguration;
+ Map<String, Map<String, MapField>> map = conf.getTypesFeaturesFieldsMapping();
+ Map<String, MapField> subMap = map.get("a-type-which-can-have-multiple-features");
+ assertEquals(2, subMap.size());
+ assertEquals("1", subMap.get("A").getFieldName(null));
+ assertEquals("2", subMap.get("B").getFieldName(null));
}
@Test
@@ -83,7 +105,7 @@ public class UIMAUpdateRequestProcessorT
assertU(commit());
assertQ(req("sentence:*"), "//*[@numFound='1']");
assertQ(req("sentiment:*"), "//*[@numFound='0']");
- assertQ(req("entity:Prague"), "//*[@numFound='1']");
+ assertQ(req("OTHER_sm:Prague"), "//*[@numFound='1']");
}
@Test
@@ -103,7 +125,7 @@ public class UIMAUpdateRequestProcessorT
assertQ(req("sentence:*"), "//*[@numFound='2']");
assertQ(req("sentiment:positive"), "//*[@numFound='1']");
- assertQ(req("entity:Apache"), "//*[@numFound='2']");
+ assertQ(req("ORGANIZATION_sm:Apache"), "//*[@numFound='2']");
}
private void addDoc(String doc) throws Exception {
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/an/DummyEntityAnnotator.java Fri May 13 11:18:19 2011
@@ -34,6 +34,12 @@ public class DummyEntityAnnotator extend
EntityAnnotation entityAnnotation = new EntityAnnotation(jcas);
entityAnnotation.setBegin(annotation.getBegin());
entityAnnotation.setEnd(annotation.getEnd());
+ String entityString = annotation.getCoveredText();
+ entityAnnotation.setEntity(entityString);
+ String name = "OTHER"; // "OTHER" makes no sense. In practice, "PERSON", "COUNTRY", "E-MAIL", etc.
+ if(entityString.equals("Apache"))
+ name = "ORGANIZATION";
+ entityAnnotation.setName(name);
entityAnnotation.addToIndexes();
}
}
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation.java Fri May 13 11:18:19 2011
@@ -1,6 +1,6 @@
-/* First created by JCasGen Fri Mar 04 12:48:08 CET 2011 */
+/* First created by JCasGen Sat May 07 22:33:38 JST 2011 */
package org.apache.solr.uima.ts;
import org.apache.uima.jcas.JCas;
@@ -11,8 +11,8 @@ import org.apache.uima.jcas.tcas.Annotat
/**
- * Updated by JCasGen Fri Mar 04 12:50:14 CET 2011
- * XML source: /Users/tommasoteofili/Documents/workspaces/lucene_workspace/lucene_dev/solr/contrib/uima/src/test/resources/DummyEntityAEDescriptor.xml
+ * Updated by JCasGen Sat May 07 22:33:38 JST 2011
+ * XML source: /Users/koji/Documents/workspace/DummyEntityAnnotator/desc/DummyEntityAEDescriptor.xml
* @generated */
public class EntityAnnotation extends Annotation {
/** @generated
@@ -57,6 +57,42 @@ public class EntityAnnotation extends An
@generated modifiable */
private void readObject() {}
-}
+
+
+ //*--------------*
+ //* Feature: name
+
+ /** getter for name - gets
+ * @generated */
+ public String getName() {
+ if (EntityAnnotation_Type.featOkTst && ((EntityAnnotation_Type)jcasType).casFeat_name == null)
+ jcasType.jcas.throwFeatMissing("name", "org.apache.solr.uima.ts.EntityAnnotation");
+ return jcasType.ll_cas.ll_getStringValue(addr, ((EntityAnnotation_Type)jcasType).casFeatCode_name);}
+
+ /** setter for name - sets
+ * @generated */
+ public void setName(String v) {
+ if (EntityAnnotation_Type.featOkTst && ((EntityAnnotation_Type)jcasType).casFeat_name == null)
+ jcasType.jcas.throwFeatMissing("name", "org.apache.solr.uima.ts.EntityAnnotation");
+ jcasType.ll_cas.ll_setStringValue(addr, ((EntityAnnotation_Type)jcasType).casFeatCode_name, v);}
+
+
+ //*--------------*
+ //* Feature: entity
+
+ /** getter for entity - gets
+ * @generated */
+ public String getEntity() {
+ if (EntityAnnotation_Type.featOkTst && ((EntityAnnotation_Type)jcasType).casFeat_entity == null)
+ jcasType.jcas.throwFeatMissing("entity", "org.apache.solr.uima.ts.EntityAnnotation");
+ return jcasType.ll_cas.ll_getStringValue(addr, ((EntityAnnotation_Type)jcasType).casFeatCode_entity);}
+
+ /** setter for entity - sets
+ * @generated */
+ public void setEntity(String v) {
+ if (EntityAnnotation_Type.featOkTst && ((EntityAnnotation_Type)jcasType).casFeat_entity == null)
+ jcasType.jcas.throwFeatMissing("entity", "org.apache.solr.uima.ts.EntityAnnotation");
+ jcasType.ll_cas.ll_setStringValue(addr, ((EntityAnnotation_Type)jcasType).casFeatCode_entity, v);}
+ }
\ No newline at end of file
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation_Type.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation_Type.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation_Type.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/java/org/apache/solr/uima/ts/EntityAnnotation_Type.java Fri May 13 11:18:19 2011
@@ -1,5 +1,5 @@
-/* First created by JCasGen Fri Mar 04 12:48:08 CET 2011 */
+/* First created by JCasGen Sat May 07 22:33:38 JST 2011 */
package org.apache.solr.uima.ts;
import org.apache.uima.jcas.JCas;
@@ -9,10 +9,12 @@ import org.apache.uima.cas.impl.FSGenera
import org.apache.uima.cas.FeatureStructure;
import org.apache.uima.cas.impl.TypeImpl;
import org.apache.uima.cas.Type;
+import org.apache.uima.cas.impl.FeatureImpl;
+import org.apache.uima.cas.Feature;
import org.apache.uima.jcas.tcas.Annotation_Type;
/**
- * Updated by JCasGen Fri Mar 04 12:50:14 CET 2011
+ * Updated by JCasGen Sat May 07 22:33:38 JST 2011
* @generated */
public class EntityAnnotation_Type extends Annotation_Type {
/** @generated */
@@ -38,6 +40,42 @@ public class EntityAnnotation_Type exten
/** @generated
@modifiable */
public final static boolean featOkTst = JCasRegistry.getFeatOkTst("org.apache.solr.uima.ts.EntityAnnotation");
+
+ /** @generated */
+ final Feature casFeat_name;
+ /** @generated */
+ final int casFeatCode_name;
+ /** @generated */
+ public String getName(int addr) {
+ if (featOkTst && casFeat_name == null)
+ jcas.throwFeatMissing("name", "org.apache.solr.uima.ts.EntityAnnotation");
+ return ll_cas.ll_getStringValue(addr, casFeatCode_name);
+ }
+ /** @generated */
+ public void setName(int addr, String v) {
+ if (featOkTst && casFeat_name == null)
+ jcas.throwFeatMissing("name", "org.apache.solr.uima.ts.EntityAnnotation");
+ ll_cas.ll_setStringValue(addr, casFeatCode_name, v);}
+
+
+
+ /** @generated */
+ final Feature casFeat_entity;
+ /** @generated */
+ final int casFeatCode_entity;
+ /** @generated */
+ public String getEntity(int addr) {
+ if (featOkTst && casFeat_entity == null)
+ jcas.throwFeatMissing("entity", "org.apache.solr.uima.ts.EntityAnnotation");
+ return ll_cas.ll_getStringValue(addr, casFeatCode_entity);
+ }
+ /** @generated */
+ public void setEntity(int addr, String v) {
+ if (featOkTst && casFeat_entity == null)
+ jcas.throwFeatMissing("entity", "org.apache.solr.uima.ts.EntityAnnotation");
+ ll_cas.ll_setStringValue(addr, casFeatCode_entity, v);}
+
+
@@ -47,6 +85,14 @@ public class EntityAnnotation_Type exten
super(jcas, casType);
casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl)this.casType, getFSGenerator());
+
+ casFeat_name = jcas.getRequiredFeatureDE(casType, "name", "uima.cas.String", featOkTst);
+ casFeatCode_name = (null == casFeat_name) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_name).getCode();
+
+
+ casFeat_entity = jcas.getRequiredFeatureDE(casType, "entity", "uima.cas.String", featOkTst);
+ casFeatCode_entity = (null == casFeat_entity) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_entity).getCode();
+
}
}
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/DummyEntityAEDescriptor.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/DummyEntityAEDescriptor.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/DummyEntityAEDescriptor.xml (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/DummyEntityAEDescriptor.xml Fri May 13 11:18:19 2011
@@ -32,6 +32,18 @@
<name>org.apache.solr.uima.ts.EntityAnnotation</name>
<description/>
<supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ <featureDescription>
+ <name>name</name>
+ <description/>
+ <rangeTypeName>uima.cas.String</rangeTypeName>
+ </featureDescription>
+ <featureDescription>
+ <name>entity</name>
+ <description/>
+ <rangeTypeName>uima.cas.String</rangeTypeName>
+ </featureDescription>
+ </features>
</typeDescription>
</types>
</typeSystemDescription>
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml Fri May 13 11:18:19 2011
@@ -597,6 +597,7 @@
stored="true" multiValued="true"/> <dynamicField name="random_*"
type="random" />
-->
+ <dynamicField name="*_sm" type="string" indexed="true" stored="true" multiValued="true"/>
<!--
uncomment the following to ignore any fields that don't already
match an existing field name or dynamic field, rather than
Modified: lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml (original)
+++ lucene/dev/branches/docvalues/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml Fri May 13 11:18:19 2011
@@ -978,20 +978,27 @@
</arr>
</lst>
<lst name="fieldMappings">
- <lst name="mapping">
- <str name="type">org.apache.uima.SentenceAnnotation</str>
- <str name="feature">coveredText</str>
- <str name="field">sentence</str>
+ <lst name="type">
+ <str name="name">org.apache.uima.SentenceAnnotation</str>
+ <lst name="mapping">
+ <str name="feature">coveredText</str>
+ <str name="field">sentence</str>
+ </lst>
</lst>
- <lst name="mapping">
- <str name="type">org.apache.solr.uima.ts.SentimentAnnotation</str>
- <str name="feature">mood</str>
- <str name="field">sentiment</str>
+ <lst name="type">
+ <str name="name">org.apache.solr.uima.ts.SentimentAnnotation</str>
+ <lst name="mapping">
+ <str name="feature">mood</str>
+ <str name="field">sentiment</str>
+ </lst>
</lst>
- <lst name="mapping">
- <str name="type">org.apache.solr.uima.ts.EntityAnnotation</str>
- <str name="feature">coveredText</str>
- <str name="field">entity</str>
+ <lst name="type">
+ <str name="name">org.apache.solr.uima.ts.EntityAnnotation</str>
+ <lst name="mapping">
+ <str name="feature">entity</str>
+ <str name="fieldNameFeature">name</str>
+ <str name="dynamicField">*_sm</str>
+ </lst>
</lst>
</lst>
</lst>
@@ -1000,6 +1007,36 @@
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
+ <updateRequestProcessorChain name="uima-multi-map">
+ <processor class="org.apache.solr.uima.processor.UIMAUpdateRequestProcessorFactory">
+ <lst name="uimaConfig">
+ <lst name="runtimeParameters">
+ <int name="ngramsize">3</int>
+ </lst>
+ <str name="analysisEngine">/TestAE.xml</str>
+ <lst name="analyzeFields">
+ <bool name="merge">false</bool>
+ <arr name="fields">
+ <str>text</str>
+ </arr>
+ </lst>
+ <lst name="fieldMappings">
+ <lst name="type">
+ <str name="name">a-type-which-can-have-multiple-features</str>
+ <lst name="mapping">
+ <str name="feature">A</str>
+ <str name="field">1</str>
+ </lst>
+ <lst name="mapping">
+ <str name="feature">B</str>
+ <str name="field">2</str>
+ </lst>
+ </lst>
+ </lst>
+ </lst>
+ </processor>
+ </updateRequestProcessorChain>
+
<!--
queryResponseWriter plugins... query responses will be written using
the writer specified by the 'wt' request parameter matching the name
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/books.json
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/books.json?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/books.json (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/books.json Fri May 13 11:18:19 2011
@@ -1,7 +1,5 @@
-{
-
-"add": {
- "doc": {
+[
+ {
"id" : "978-0641723445",
"cat" : ["book","hardcover"],
"title" : "The Lightning Thief",
@@ -13,11 +11,8 @@
"price" : 12.50,
"pages_i" : 384
}
-}
-
,
-"add": {
- "doc": {
+ {
"id" : "978-1423103349",
"cat" : ["book","paperback"],
"title" : "The Sea of Monsters",
@@ -29,6 +24,4 @@
"price" : 6.49,
"pages_i" : 304
}
-}
-
-}
+]
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/hd.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/hd.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/hd.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/hd.xml Fri May 13 11:18:19 2011
@@ -20,6 +20,8 @@
<field name="id">SP2514N</field>
<field name="name">Samsung SpinPoint P120 SP2514N - hard drive - 250 GB - ATA-133</field>
<field name="manu">Samsung Electronics Co. Ltd.</field>
+ <!-- Join -->
+ <field name="manu_id_s">samsung</field>
<field name="cat">electronics</field>
<field name="cat">hard drive</field>
<field name="features">7200RPM, 8MB cache, IDE Ultra ATA-133</field>
@@ -36,6 +38,8 @@
<field name="id">6H500F0</field>
<field name="name">Maxtor DiamondMax 11 - hard drive - 500 GB - SATA-300</field>
<field name="manu">Maxtor Corp.</field>
+ <!-- Join -->
+ <field name="manu_id_s">maxtor</field>
<field name="cat">electronics</field>
<field name="cat">hard drive</field>
<field name="features">SATA 3.0Gb/s, NCQ</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_other.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_other.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_other.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_other.xml Fri May 13 11:18:19 2011
@@ -21,6 +21,8 @@
<field name="id">F8V7067-APL-KIT</field>
<field name="name">Belkin Mobile Power Cord for iPod w/ Dock</field>
<field name="manu">Belkin</field>
+ <!-- Join -->
+ <field name="manu_id_s">belkin</field>
<field name="cat">electronics</field>
<field name="cat">connector</field>
<field name="features">car power adapter, white</field>
@@ -37,6 +39,8 @@
<field name="id">IW-02</field>
<field name="name">iPod & iPod Mini USB 2.0 Cable</field>
<field name="manu">Belkin</field>
+ <!-- Join -->
+ <field name="manu_id_s">belkin</field>
<field name="cat">electronics</field>
<field name="cat">connector</field>
<field name="features">car power adapter for iPod, white</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_video.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_video.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_video.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/ipod_video.xml Fri May 13 11:18:19 2011
@@ -19,6 +19,8 @@
<field name="id">MA147LL/A</field>
<field name="name">Apple 60 GB iPod with Video Playback Black</field>
<field name="manu">Apple Computer Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">apple</field>
<field name="cat">electronics</field>
<field name="cat">music</field>
<field name="features">iTunes, Podcasts, Audiobooks</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/mem.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/mem.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/mem.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/mem.xml Fri May 13 11:18:19 2011
@@ -20,6 +20,8 @@
<field name="id">TWINX2048-3200PRO</field>
<field name="name">CORSAIR XMS 2GB (2 x 1GB) 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) Dual Channel Kit System Memory - Retail</field>
<field name="manu">Corsair Microsystems Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">corsair</field>
<field name="cat">electronics</field>
<field name="cat">memory</field>
<field name="features">CAS latency 2, 2-3-3-6 timing, 2.75v, unbuffered, heat-spreader</field>
@@ -38,6 +40,8 @@
<field name="id">VS1GB400C3</field>
<field name="name">CORSAIR ValueSelect 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - Retail</field>
<field name="manu">Corsair Microsystems Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">corsair</field>
<field name="cat">electronics</field>
<field name="cat">memory</field>
<field name="price">74.99</field>
@@ -54,6 +58,8 @@
<field name="id">VDBDB1A16</field>
<field name="name">A-DATA V-Series 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - OEM</field>
<field name="manu">A-DATA Technology Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">corsair</field>
<field name="cat">electronics</field>
<field name="cat">memory</field>
<field name="features">CAS latency 3, 2.7v</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/monitor.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/monitor.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/monitor.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/monitor.xml Fri May 13 11:18:19 2011
@@ -19,6 +19,8 @@
<field name="id">3007WFP</field>
<field name="name">Dell Widescreen UltraSharp 3007WFP</field>
<field name="manu">Dell, Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">dell</field>
<field name="cat">electronics</field>
<field name="cat">monitor</field>
<field name="features">30" TFT active matrix LCD, 2560 x 1600, .25mm dot pitch, 700:1 contrast</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/monitor2.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/monitor2.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/monitor2.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/monitor2.xml Fri May 13 11:18:19 2011
@@ -19,6 +19,8 @@
<field name="id">VA902B</field>
<field name="name">ViewSonic VA902B - flat panel display - TFT - 19"</field>
<field name="manu">ViewSonic Corp.</field>
+ <!-- Join -->
+ <field name="manu_id_s">viewsonic</field>
<field name="cat">electronics</field>
<field name="cat">monitor</field>
<field name="features">19" TFT active matrix LCD, 8ms response time, 1280 x 1024 native resolution</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/mp500.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/mp500.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/mp500.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/mp500.xml Fri May 13 11:18:19 2011
@@ -19,6 +19,8 @@
<field name="id">0579B002</field>
<field name="name">Canon PIXMA MP500 All-In-One Photo Printer</field>
<field name="manu">Canon Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">canon</field>
<field name="cat">electronics</field>
<field name="cat">multifunction printer</field>
<field name="cat">printer</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/sd500.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/sd500.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/sd500.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/sd500.xml Fri May 13 11:18:19 2011
@@ -19,6 +19,8 @@
<field name="id">9885A004</field>
<field name="name">Canon PowerShot SD500</field>
<field name="manu">Canon Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">canon</field>
<field name="cat">electronics</field>
<field name="cat">camera</field>
<field name="features">3x zoop, 7.1 megapixel Digital ELPH</field>
Modified: lucene/dev/branches/docvalues/solr/example/exampledocs/vidcard.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/exampledocs/vidcard.xml?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/exampledocs/vidcard.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/exampledocs/vidcard.xml Fri May 13 11:18:19 2011
@@ -19,7 +19,10 @@
<doc>
<field name="id">EN7800GTX/2DHTV/256M</field>
<field name="name">ASUS Extreme N7800GTX/2DHTV (256 MB)</field>
+ <!-- Denormalized -->
<field name="manu">ASUS Computer Inc.</field>
+ <!-- Join -->
+ <field name="manu_id_s">asus</field>
<field name="cat">electronics</field>
<field name="cat">graphics card</field>
<field name="features">NVIDIA GeForce 7800 GTX GPU/VPU clocked at 486MHz</field>
@@ -39,6 +42,8 @@
<field name="id">100-435805</field>
<field name="name">ATI Radeon X1900 XTX 512 MB PCIE Video Card</field>
<field name="manu">ATI Technologies</field>
+ <!-- Join -->
+ <field name="manu_id_s">ati</field>
<field name="cat">electronics</field>
<field name="cat">graphics card</field>
<field name="features">ATI RADEON X1900 GPU/VPU clocked at 650MHz</field>
Modified: lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/doc.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/doc.vm?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/doc.vm (original)
+++ lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/doc.vm Fri May 13 11:18:19 2011
@@ -26,4 +26,17 @@
#if($params.getBool("debugQuery",false))
<a href="#" onclick='jQuery(this).siblings("pre").toggle(); return false;'>toggle explain</a>
<pre style="display:none">$response.getExplainMap().get($doc.getFirstValue('id'))</pre>
+ <a href="#" onclick='jQuery(this).siblings("pre2").toggle(); return false;'>toggle all fields</a>
+ <pre2 style="display:none">
+ #foreach($fieldname in $doc.fieldNames)
+ <br>
+ <span class="field-name">$fieldname :</span>
+ <span>
+ #foreach($value in $doc.getFieldValues($fieldname))
+ $value
+ #end
+ </span>
+ #end
+ </br>
+ </pre2>
#end
\ No newline at end of file
Modified: lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/footer.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/footer.vm?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/footer.vm (original)
+++ lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/footer.vm Fri May 13 11:18:19 2011
@@ -4,7 +4,7 @@
#if($request.params.get('debugQuery'))
<a href="#url_for_home?#q#if($list.size($request.params.getParams('fq')) > 0)&#fqs($request.params.getParams('fq'))#end">disable debug</a>
#else
- <a href="#url_for_lens&debugQuery=true">enable debug</a>
+ <a href="#url_for_lens&debugQuery=true&fl=*,score">enable debug</a>
#end
#if($annotate)
<a href="#url_for_home?#q#if($list.size($request.params.getParams('fq')) > 0)&#fqs($request.params.getParams('fq'))#end#boostPrice">disable annotation</a>
Modified: lucene/dev/branches/docvalues/solr/lib/apache-solr-noggit-pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/lib/apache-solr-noggit-pom.xml.template?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/lib/apache-solr-noggit-pom.xml.template (original)
+++ lucene/dev/branches/docvalues/solr/lib/apache-solr-noggit-pom.xml.template Fri May 13 11:18:19 2011
@@ -31,6 +31,6 @@
<artifactId>solr-noggit</artifactId>
<name>Solr Specific Noggit</name>
<version>@version@</version>
- <description>Solr Specific Noggit r944541</description>
+ <description>Solr Specific Noggit r1099557</description>
<packaging>jar</packaging>
</project>
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/PathHierarchyTokenizerFactory.java Fri May 13 11:18:19 2011
@@ -21,6 +21,7 @@ import java.util.Map;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
+import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
/**
@@ -37,6 +38,8 @@ public class PathHierarchyTokenizerFacto
private char delimiter;
private char replacement;
+ private boolean reverse = false;
+ private int skip = PathHierarchyTokenizer.DEFAULT_SKIP;
/**
* Require a configured pattern
@@ -70,10 +73,23 @@ public class PathHierarchyTokenizerFacto
else{
replacement = delimiter;
}
+
+ v = args.get( "reverse" );
+ if( v != null ){
+ reverse = "true".equals( v );
+ }
+
+ v = args.get( "skip" );
+ if( v != null ){
+ skip = Integer.parseInt( v );
+ }
}
public Tokenizer create(Reader input) {
- return new PathHierarchyTokenizer(input, delimiter, replacement);
+ if( reverse ) {
+ return new ReversePathHierarchyTokenizer(input, delimiter, replacement, skip);
+ }
+ return new PathHierarchyTokenizer(input, delimiter, replacement, skip);
}
}
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/SynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/SynonymFilterFactory.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/SynonymFilterFactory.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/SynonymFilterFactory.java Fri May 13 11:18:19 2011
@@ -61,6 +61,16 @@ public class SynonymFilterFactory extend
tokFactory = loadTokenizerFactory( loader, tf, args );
}
+ Iterable<String> wlist=loadRules( synonyms, loader );
+
+ synMap = new SynonymMap(ignoreCase);
+ parseRules(wlist, synMap, "=>", ",", expand,tokFactory);
+ }
+
+ /**
+ * @return a list of all rules
+ */
+ protected Iterable<String> loadRules( String synonyms, ResourceLoader loader ) {
List<String> wlist=null;
try {
File synonymFile = new File(synonyms);
@@ -77,13 +87,12 @@ public class SynonymFilterFactory extend
} catch (IOException e) {
throw new RuntimeException(e);
}
- synMap = new SynonymMap(ignoreCase);
- parseRules(wlist, synMap, "=>", ",", expand,tokFactory);
+ return wlist;
}
private SynonymMap synMap;
- static void parseRules(List<String> rules, SynonymMap map, String mappingSep,
+ static void parseRules(Iterable<String> rules, SynonymMap map, String mappingSep,
String synSep, boolean expansion, TokenizerFactory tokFactory) {
int count=0;
for (String rule : rules) {
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonLoader.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonLoader.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonLoader.java Fri May 13 11:18:19 2011
@@ -23,6 +23,7 @@ import java.util.Stack;
import org.apache.commons.io.IOUtils;
import org.apache.noggit.JSONParser;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.ContentStream;
@@ -43,10 +44,18 @@ import org.slf4j.LoggerFactory;
class JsonLoader extends ContentStreamLoader {
final static Logger log = LoggerFactory.getLogger( JsonLoader.class );
- protected UpdateRequestProcessor processor;
+ protected final UpdateRequestProcessor processor;
+ protected final SolrQueryRequest req;
+ protected JSONParser parser;
+ protected final int commitWithin;
+ protected final boolean overwrite;
- public JsonLoader(UpdateRequestProcessor processor) {
+ public JsonLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
this.processor = processor;
+ this.req = req;
+
+ commitWithin = req.getParams().getInt(XmlUpdateRequestHandler.COMMIT_WITHIN, -1);
+ overwrite = req.getParams().getBool(XmlUpdateRequestHandler.OVERWRITE, true);
}
@Override
@@ -55,14 +64,14 @@ class JsonLoader extends ContentStreamLo
Reader reader = null;
try {
reader = stream.getReader();
- if (XmlUpdateRequestHandler.log.isTraceEnabled()) {
+ if (log.isTraceEnabled()) {
String body = IOUtils.toString(reader);
- XmlUpdateRequestHandler.log.trace("body", body);
+ log.trace("body", body);
reader = new StringReader(body);
}
- JSONParser parser = new JSONParser(reader);
- this.processUpdate(req, processor, parser);
+ parser = new JSONParser(reader);
+ this.processUpdate();
}
finally {
IOUtils.closeQuietly(reader);
@@ -70,39 +79,50 @@ class JsonLoader extends ContentStreamLo
}
@SuppressWarnings("fallthrough")
- void processUpdate(SolrQueryRequest req, UpdateRequestProcessor processor, JSONParser parser) throws IOException
+ void processUpdate() throws IOException
{
int ev = parser.nextEvent();
while( ev != JSONParser.EOF ) {
switch( ev )
{
+ case JSONParser.ARRAY_START:
+ handleAdds();
+ break;
+
case JSONParser.STRING:
if( parser.wasKey() ) {
String v = parser.getString();
if( v.equals( XmlUpdateRequestHandler.ADD ) ) {
- processor.processAdd( parseAdd(req, parser ) );
+ int ev2 = parser.nextEvent();
+ if (ev2 == JSONParser.OBJECT_START) {
+ processor.processAdd( parseAdd() );
+ } else if (ev2 == JSONParser.ARRAY_START) {
+ handleAdds();
+ } else {
+ assertEvent(ev2, JSONParser.OBJECT_START);
+ }
}
else if( v.equals( XmlUpdateRequestHandler.COMMIT ) ) {
CommitUpdateCommand cmd = new CommitUpdateCommand(req, false );
cmd.waitFlush = cmd.waitSearcher = true;
- parseCommitOptions( parser, cmd );
+ parseCommitOptions( cmd );
processor.processCommit( cmd );
}
else if( v.equals( XmlUpdateRequestHandler.OPTIMIZE ) ) {
CommitUpdateCommand cmd = new CommitUpdateCommand(req, true );
cmd.waitFlush = cmd.waitSearcher = true;
- parseCommitOptions( parser, cmd );
+ parseCommitOptions( cmd );
processor.processCommit( cmd );
}
else if( v.equals( XmlUpdateRequestHandler.DELETE ) ) {
- processor.processDelete( parseDelete(req, parser ) );
+ processor.processDelete( parseDelete() );
}
else if( v.equals( XmlUpdateRequestHandler.ROLLBACK ) ) {
- processor.processRollback( parseRollback(req, parser ) );
+ processor.processRollback( parseRollback() );
}
else {
- throw new IOException( "Unknown command: "+v+" ["+parser.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown command: "+v+" ["+parser.getPosition()+"]" );
}
break;
}
@@ -117,12 +137,11 @@ class JsonLoader extends ContentStreamLo
case JSONParser.OBJECT_START:
case JSONParser.OBJECT_END:
- case JSONParser.ARRAY_START:
case JSONParser.ARRAY_END:
break;
default:
- System.out.println("UNKNOWN_EVENT_ID:"+ev);
+ log.info("Noggit UNKNOWN_EVENT_ID:"+ev);
break;
}
// read the next event
@@ -130,187 +149,211 @@ class JsonLoader extends ContentStreamLo
}
}
- DeleteUpdateCommand parseDelete(SolrQueryRequest req, JSONParser js) throws IOException {
- assertNextEvent( js, JSONParser.OBJECT_START );
+ DeleteUpdateCommand parseDelete() throws IOException {
+ assertNextEvent( JSONParser.OBJECT_START );
DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
-
+
while( true ) {
- int ev = js.nextEvent();
+ int ev = parser.nextEvent();
if( ev == JSONParser.STRING ) {
- String key = js.getString();
- if( js.wasKey() ) {
+ String key = parser.getString();
+ if( parser.wasKey() ) {
if( "id".equals( key ) ) {
- cmd.id = js.getString();
+ cmd.id = parser.getString();
}
else if( "query".equals(key) ) {
- cmd.query = js.getString();
+ cmd.query = parser.getString();
}
else {
- throw new IOException( "Unknown key: "+key+" ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" );
}
}
else {
- throw new IOException(
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"invalid string: " + key
- +" at ["+js.getPosition()+"]" );
+ +" at ["+parser.getPosition()+"]" );
}
}
else if( ev == JSONParser.OBJECT_END ) {
if( cmd.id == null && cmd.query == null ) {
- throw new IOException( "Missing id or query for delete ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing id or query for delete ["+parser.getPosition()+"]" );
}
return cmd;
}
else {
- throw new IOException(
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Got: "+JSONParser.getEventString( ev )
- +" at ["+js.getPosition()+"]" );
+ +" at ["+parser.getPosition()+"]" );
}
}
}
- RollbackUpdateCommand parseRollback(SolrQueryRequest req, JSONParser js) throws IOException {
- assertNextEvent( js, JSONParser.OBJECT_START );
- assertNextEvent( js, JSONParser.OBJECT_END );
+ RollbackUpdateCommand parseRollback() throws IOException {
+ assertNextEvent( JSONParser.OBJECT_START );
+ assertNextEvent( JSONParser.OBJECT_END );
return new RollbackUpdateCommand(req);
}
- void parseCommitOptions( JSONParser js, CommitUpdateCommand cmd ) throws IOException
+ void parseCommitOptions(CommitUpdateCommand cmd ) throws IOException
{
- assertNextEvent( js, JSONParser.OBJECT_START );
+ assertNextEvent( JSONParser.OBJECT_START );
while( true ) {
- int ev = js.nextEvent();
+ int ev = parser.nextEvent();
if( ev == JSONParser.STRING ) {
- String key = js.getString();
- if( js.wasKey() ) {
+ String key = parser.getString();
+ if( parser.wasKey() ) {
if( XmlUpdateRequestHandler.WAIT_SEARCHER.equals( key ) ) {
- cmd.waitSearcher = js.getBoolean();
+ cmd.waitSearcher = parser.getBoolean();
}
else if( XmlUpdateRequestHandler.WAIT_FLUSH.equals( key ) ) {
- cmd.waitFlush = js.getBoolean();
+ cmd.waitFlush = parser.getBoolean();
}
else {
- throw new IOException( "Unknown key: "+key+" ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" );
}
}
else {
- throw new IOException(
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"invalid string: " + key
- +" at ["+js.getPosition()+"]" );
+ +" at ["+parser.getPosition()+"]" );
}
}
else if( ev == JSONParser.OBJECT_END ) {
return;
}
else {
- throw new IOException(
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Got: "+JSONParser.getEventString( ev )
- +" at ["+js.getPosition()+"]" );
+ +" at ["+parser.getPosition()+"]" );
}
}
}
- AddUpdateCommand parseAdd(SolrQueryRequest req, JSONParser js ) throws IOException
+ AddUpdateCommand parseAdd() throws IOException
{
- assertNextEvent( js, JSONParser.OBJECT_START );
AddUpdateCommand cmd = new AddUpdateCommand(req);
+ cmd.commitWithin = commitWithin;
+ cmd.overwrite = overwrite;
+
float boost = 1.0f;
while( true ) {
- int ev = js.nextEvent();
+ int ev = parser.nextEvent();
if( ev == JSONParser.STRING ) {
- if( js.wasKey() ) {
- String key = js.getString();
+ if( parser.wasKey() ) {
+ String key = parser.getString();
if( "doc".equals( key ) ) {
if( cmd.solrDoc != null ) {
- throw new IOException( "multiple docs in same add command" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "multiple docs in same add command" );
}
- ev = assertNextEvent( js, JSONParser.OBJECT_START );
- cmd.solrDoc = parseDoc( ev, js );
+ ev = assertNextEvent( JSONParser.OBJECT_START );
+ cmd.solrDoc = parseDoc( ev );
}
else if( XmlUpdateRequestHandler.OVERWRITE.equals( key ) ) {
- cmd.overwrite = js.getBoolean(); // reads next boolean
+ cmd.overwrite = parser.getBoolean(); // reads next boolean
}
else if( XmlUpdateRequestHandler.COMMIT_WITHIN.equals( key ) ) {
- cmd.commitWithin = (int)js.getLong();
+ cmd.commitWithin = (int)parser.getLong();
}
else if( "boost".equals( key ) ) {
- boost = Float.parseFloat( js.getNumberChars().toString() );
+ boost = Float.parseFloat( parser.getNumberChars().toString() );
}
else {
- throw new IOException( "Unknown key: "+key+" ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" );
}
}
else {
- throw new IOException(
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Should be a key "
- +" at ["+js.getPosition()+"]" );
+ +" at ["+parser.getPosition()+"]" );
}
}
else if( ev == JSONParser.OBJECT_END ) {
if( cmd.solrDoc == null ) {
- throw new IOException("missing solr document. "+js.getPosition() );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"missing solr document. "+parser.getPosition() );
}
cmd.solrDoc.setDocumentBoost( boost );
return cmd;
}
else {
- throw new IOException(
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Got: "+JSONParser.getEventString( ev )
- +" at ["+js.getPosition()+"]" );
+ +" at ["+parser.getPosition()+"]" );
}
}
}
-
- int assertNextEvent( JSONParser parser, int ev ) throws IOException
+
+
+ void handleAdds() throws IOException
+ {
+ while( true ) {
+ AddUpdateCommand cmd = new AddUpdateCommand(req);
+ cmd.commitWithin = commitWithin;
+ cmd.overwrite = overwrite;
+
+ int ev = parser.nextEvent();
+ if (ev == JSONParser.ARRAY_END) break;
+
+ assertEvent(ev, JSONParser.OBJECT_START);
+ cmd.solrDoc = parseDoc(ev);
+ processor.processAdd(cmd);
+ }
+ }
+
+
+ int assertNextEvent(int expected ) throws IOException
{
int got = parser.nextEvent();
- if( ev != got ) {
- throw new IOException(
- "Expected: "+JSONParser.getEventString( ev )
- +" but got "+JSONParser.getEventString( got )
+ assertEvent(got, expected);
+ return got;
+ }
+
+ void assertEvent(int ev, int expected) {
+ if( ev != expected ) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Expected: "+JSONParser.getEventString( expected )
+ +" but got "+JSONParser.getEventString( ev )
+" at ["+parser.getPosition()+"]" );
}
- return got;
}
- SolrInputDocument parseDoc( int ev, JSONParser js ) throws IOException
+ SolrInputDocument parseDoc(int ev) throws IOException
{
Stack<Object> stack = new Stack<Object>();
Object obj = null;
boolean inArray = false;
if( ev != JSONParser.OBJECT_START ) {
- throw new IOException( "object should already be started" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "object should already be started" );
}
while( true ) {
- //System.out.println( ev + "["+JSONParser.getEventString(ev)+"] "+js.wasKey() ); //+ js.getString() );
+ //System.out.println( ev + "["+JSONParser.getEventString(ev)+"] "+parser.wasKey() ); //+ parser.getString() );
switch (ev) {
case JSONParser.STRING:
- if( js.wasKey() ) {
+ if( parser.wasKey() ) {
obj = stack.peek();
- String v = js.getString();
+ String v = parser.getString();
if( obj instanceof SolrInputField ) {
SolrInputField field = (SolrInputField)obj;
if( "boost".equals( v ) ) {
- ev = js.nextEvent();
+ ev = parser.nextEvent();
if( ev != JSONParser.NUMBER &&
ev != JSONParser.LONG &&
ev != JSONParser.BIGNUMBER ) {
- throw new IOException( "boost should have number! "+JSONParser.getEventString(ev) );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "boost should have number! "+JSONParser.getEventString(ev) );
}
- field.setBoost( Float.valueOf( js.getNumberChars().toString() ) );
+ field.setBoost( Float.valueOf( parser.getNumberChars().toString() ) );
}
else if( "value".equals( v ) ) {
// nothing special...
stack.push( field ); // so it can be popped
}
else {
- throw new IOException( "invalid key: "+v + " ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "invalid key: "+v + " ["+ parser.getPosition()+"]" );
}
}
else if( obj instanceof SolrInputDocument ) {
@@ -323,22 +366,22 @@ class JsonLoader extends ContentStreamLo
stack.push( f );
}
else {
- throw new IOException( "hymmm ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "hymmm ["+ parser.getPosition()+"]" );
}
}
else {
- addValToField(stack, js.getString(), inArray, js);
+ addValToField(stack, parser.getString(), inArray, parser);
}
break;
case JSONParser.LONG:
case JSONParser.NUMBER:
case JSONParser.BIGNUMBER:
- addValToField(stack, js.getNumberChars().toString(), inArray, js);
+ addValToField(stack, parser.getNumberChars().toString(), inArray, parser);
break;
case JSONParser.BOOLEAN:
- addValToField(stack, js.getBoolean(),inArray, js);
+ addValToField(stack, parser.getBoolean(),inArray, parser);
break;
case JSONParser.OBJECT_START:
@@ -351,7 +394,7 @@ class JsonLoader extends ContentStreamLo
// should alreay be pushed...
}
else {
- throw new IOException( "should not start new object with: "+obj + " ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should not start new object with: "+obj + " ["+ parser.getPosition()+"]" );
}
}
break;
@@ -365,7 +408,7 @@ class JsonLoader extends ContentStreamLo
// should already be pushed...
}
else {
- throw new IOException( "should not start new object with: "+obj + " ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should not start new object with: "+obj + " ["+ parser.getPosition()+"]" );
}
break;
@@ -383,18 +426,18 @@ class JsonLoader extends ContentStreamLo
break;
}
- ev = js.nextEvent();
+ ev = parser.nextEvent();
if( ev == JSONParser.EOF ) {
- throw new IOException( "should finish doc first!" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should finish doc first!" );
}
}
}
- static void addValToField( Stack stack, Object val, boolean inArray, JSONParser js ) throws IOException
+ static void addValToField( Stack stack, Object val, boolean inArray, JSONParser parser ) throws IOException
{
Object obj = stack.peek();
if( !(obj instanceof SolrInputField) ) {
- throw new IOException( "hymmm ["+js.getPosition()+"]" );
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "hymmm ["+parser.getPosition()+"]" );
}
SolrInputField f = inArray
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java Fri May 13 11:18:19 2011
@@ -37,7 +37,7 @@ public class JsonUpdateRequestHandler ex
@Override
protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- return new JsonLoader(processor);
+ return new JsonLoader(req, processor);
}
//////////////////////// SolrInfoMBeans methods //////////////////////
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java Fri May 13 11:18:19 2011
@@ -208,7 +208,7 @@ public class TermVectorComponent extends
if (keyField != null) {
Document document = reader.document(docId, fieldSelector);
- Fieldable uniqId = document.getField(uniqFieldName);
+ Fieldable uniqId = document.getFieldable(uniqFieldName);
String uniqVal = null;
if (uniqId != null) {
uniqVal = keyField.getType().storedToReadable(uniqId);
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Fri May 13 11:18:19 2011
@@ -401,13 +401,24 @@ public class DefaultSolrHighlighter exte
private void doHighlightingByHighlighter( Query query, SolrQueryRequest req, NamedList docSummaries,
int docId, Document doc, String fieldName ) throws IOException {
+ final SolrIndexSearcher searcher = req.getSearcher();
+ final IndexSchema schema = searcher.getSchema();
+
+ // TODO: Currently in trunk highlighting numeric fields is broken (Lucene) -
+ // so we disable them until fixed (see LUCENE-3080)!
+ // BEGIN: Hack
+ final SchemaField schemaField = schema.getFieldOrNull(fieldName);
+ if (schemaField != null && (
+ (schemaField.getType() instanceof org.apache.solr.schema.TrieField) ||
+ (schemaField.getType() instanceof org.apache.solr.schema.TrieDateField)
+ )) return;
+ // END: Hack
+
SolrParams params = req.getParams();
String[] docTexts = doc.getValues(fieldName);
// according to Document javadoc, doc.getValues() never returns null. check empty instead of null
if (docTexts.length == 0) return;
- SolrIndexSearcher searcher = req.getSearcher();
- IndexSchema schema = searcher.getSchema();
TokenStream tstream = null;
int numFragments = getMaxSnippets(fieldName, params);
boolean mergeContiguousFragments = isMergeContiguousFragments(fieldName, params);
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/IndexSchema.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/IndexSchema.java Fri May 13 11:18:19 2011
@@ -797,19 +797,23 @@ public final class IndexSchema {
NamedNodeMap attrs = node.getAttributes();
String analyzerName = DOMUtil.getAttr(attrs,"class");
if (analyzerName != null) {
- // No need to be core-aware as Analyzers are not in the core-aware list
- final Class<? extends Analyzer> clazz = loader.findClass(analyzerName).asSubclass(Analyzer.class);
try {
+ // No need to be core-aware as Analyzers are not in the core-aware list
+ final Class<? extends Analyzer> clazz = loader.findClass
+ (analyzerName).asSubclass(Analyzer.class);
+
try {
- // first try to use a ctor with version parameter (needed for many new Analyzers that have no default one anymore)
+ // first try to use a ctor with version parameter
+ // (needed for many new Analyzers that have no default one anymore)
Constructor<? extends Analyzer> cnstr = clazz.getConstructor(Version.class);
final String matchVersionStr = DOMUtil.getAttr(attrs, LUCENE_MATCH_VERSION_PARAM);
final Version luceneMatchVersion = (matchVersionStr == null) ?
solrConfig.luceneMatchVersion : Config.parseLuceneVersionString(matchVersionStr);
if (luceneMatchVersion == null) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Configuration Error: Analyzer '" + clazz.getName() +
- "' needs a 'luceneMatchVersion' parameter");
+ throw new SolrException
+ ( SolrException.ErrorCode.SERVER_ERROR,
+ "Configuration Error: Analyzer '" + clazz.getName() +
+ "' needs a 'luceneMatchVersion' parameter");
}
return cnstr.newInstance(luceneMatchVersion);
} catch (NoSuchMethodException nsme) {
@@ -817,8 +821,9 @@ public final class IndexSchema {
return clazz.newInstance();
}
} catch (Exception e) {
+ log.error("Cannot load analyzer: "+analyzerName, e);
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Cannot load analyzer: "+analyzerName );
+ "Cannot load analyzer: "+analyzerName, e );
}
}
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/SchemaField.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/SchemaField.java Fri May 13 11:18:19 2011
@@ -19,7 +19,6 @@ package org.apache.solr.schema;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.QParser;
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java?rev=1102677&r1=1102676&r2=1102677&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java Fri May 13 11:18:19 2011
@@ -18,210 +18,125 @@
package org.apache.solr.schema;
import org.apache.noggit.CharArr;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.analysis.CharFilterFactory;
-import org.apache.solr.analysis.TokenFilterFactory;
-import org.apache.solr.analysis.TokenizerChain;
-import org.apache.solr.analysis.TrieTokenizerFactory;
-import org.apache.solr.search.function.*;
+import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.QParser;
import org.apache.solr.response.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.cache.CachedArrayCreator;
-import org.apache.lucene.search.cache.LongValuesCreator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.NumericTokenStream;
import java.util.Map;
import java.util.Date;
import java.io.IOException;
public class TrieDateField extends DateField {
- protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP; // the one passed in or defaulted
- protected int precisionStep = precisionStepArg; // normalized
+
+ final TrieField wrappedField = new TrieField() {{
+ type = TrieTypes.DATE;
+ }};
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
- String p = args.remove("precisionStep");
- if (p != null) {
- precisionStepArg = Integer.parseInt(p);
- }
- // normalize the precisionStep
- precisionStep = precisionStepArg;
- if (precisionStep<=0 || precisionStep>=64) precisionStep=Integer.MAX_VALUE;
-
- CharFilterFactory[] filterFactories = new CharFilterFactory[0];
- TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
- analyzer = new TokenizerChain(filterFactories, new TrieTokenizerFactory(TrieField.TrieTypes.DATE, precisionStep), tokenFilterFactories);
- // for query time we only need one token, so we use the biggest possible precisionStep:
- queryAnalyzer = new TokenizerChain(filterFactories, new TrieTokenizerFactory(TrieField.TrieTypes.DATE, Integer.MAX_VALUE), tokenFilterFactories);
+ wrappedField.init(schema, args);
+ analyzer = wrappedField.analyzer;
+ queryAnalyzer = wrappedField.queryAnalyzer;
}
@Override
public Date toObject(Fieldable f) {
- byte[] arr = f.getBinaryValue();
- if (arr==null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,TrieField.badFieldString(f));
- return new Date(TrieFieldHelper.toLong(arr));
+ return (Date) wrappedField.toObject(f);
}
@Override
public Object toObject(SchemaField sf, BytesRef term) {
- return new Date(NumericUtils.prefixCodedToLong(term));
+ return wrappedField.toObject(sf, term);
}
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- int flags = CachedArrayCreator.CACHE_VALUES_AND_BITS;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- Object missingValue = null;
- if( sortMissingLast ) {
- missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
- } else if( sortMissingFirst ) {
- missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
- }
- return new SortField(new LongValuesCreator(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, flags), top).setMissingValue(missingValue);
+ return wrappedField.getSortField(field, top);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser parser) {
- field.checkFieldCacheSource(parser);
- return new TrieDateFieldSource( new LongValuesCreator( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, CachedArrayCreator.CACHE_VALUES_AND_BITS ));
+ return wrappedField.getValueSource(field, parser);
+ }
+
+ /**
+ * @return the precisionStep used to index values into the field
+ */
+ public int getPrecisionStep() {
+ return wrappedField.getPrecisionStep();
}
+
@Override
public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException {
- byte[] arr = f.getBinaryValue();
- if (arr==null) {
- writer.writeStr(name, TrieField.badFieldString(f),true);
- return;
- }
-
- writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
+ wrappedField.write(writer, name, f);
}
@Override
public boolean isTokenized() {
- return true;
+ return wrappedField.isTokenized();
}
- /**
- * @return the precisionStep used to index values into the field
- */
- public int getPrecisionStep() {
- return precisionStepArg;
+ @Override
+ public boolean multiValuedFieldCache() {
+ return wrappedField.multiValuedFieldCache();
}
-
-
@Override
public String storedToReadable(Fieldable f) {
- return toExternal(f);
+ return wrappedField.storedToReadable(f);
}
@Override
public String readableToIndexed(String val) {
- // TODO: Numeric should never be handled as String, that may break in future lucene versions! Change to use BytesRef for term texts!
- BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_LONG);
- NumericUtils.longToPrefixCoded(super.parseMath(null, val).getTime(), 0, bytes);
- return bytes.utf8ToString();
+ return wrappedField.readableToIndexed(val);
}
@Override
public String toInternal(String val) {
- return readableToIndexed(val);
+ return wrappedField.toInternal(val);
}
@Override
public String toExternal(Fieldable f) {
- byte[] arr = f.getBinaryValue();
- if (arr==null) return TrieField.badFieldString(f);
- return super.toExternal(new Date(TrieFieldHelper.toLong(arr)));
+ return wrappedField.toExternal(f);
}
@Override
public String indexedToReadable(String _indexedForm) {
- final BytesRef indexedForm = new BytesRef(_indexedForm);
- return super.toExternal( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
+ return wrappedField.indexedToReadable(_indexedForm);
}
@Override
public void indexedToReadable(BytesRef input, CharArr out) {
- String ext = super.toExternal( new Date(NumericUtils.prefixCodedToLong(input)) );
- out.write(ext);
+ wrappedField.indexedToReadable(input, out);
}
@Override
public String storedToIndexed(Fieldable f) {
- // TODO: optimize to remove redundant string conversion
- return readableToIndexed(storedToReadable(f));
+ return wrappedField.storedToIndexed(f);
}
@Override
public Fieldable createField(SchemaField field, Object value, float boost) {
- boolean indexed = field.indexed();
- boolean stored = field.stored();
-
- if (!indexed && !stored) {
- if (log.isTraceEnabled())
- log.trace("Ignoring unindexed/unstored field: " + field);
- return null;
- }
-
- int ps = precisionStep;
-
- byte[] arr=null;
- TokenStream ts=null;
-
- long time = (value instanceof Date)
- ? ((Date)value).getTime()
- : super.parseMath(null, value.toString()).getTime();
-
- if (stored) arr = TrieFieldHelper.toArr(time);
- if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
-
- Field f;
- if (stored) {
- f = new Field(field.getName(), arr);
- if (indexed) f.setTokenStream(ts);
- } else {
- f = new Field(field.getName(), ts);
- }
-
- // term vectors aren't supported
-
- f.setOmitNorms(field.omitNorms());
- f.setOmitTermFreqAndPositions(field.omitTf());
- f.setBoost(boost);
- return f;
+ return wrappedField.createField(field, value, boost);
}
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
- return getRangeQuery(parser, field,
- min==null ? null : super.parseMath(null,min),
- max==null ? null : super.parseMath(null,max),
- minInclusive, maxInclusive);
+ return wrappedField.getRangeQuery(parser, field, min, max, minInclusive, maxInclusive);
}
@Override
public Query getRangeQuery(QParser parser, SchemaField sf, Date min, Date max, boolean minInclusive, boolean maxInclusive) {
- int ps = precisionStep;
- Query query = NumericRangeQuery.newLongRange(sf.getName(), ps,
+ return NumericRangeQuery.newLongRange(sf.getName(), wrappedField.precisionStep,
min == null ? null : min.getTime(),
max == null ? null : max.getTime(),
minInclusive, maxInclusive);
-
- return query;
}
}