You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/11/20 08:19:41 UTC
svn commit: r719187 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Wed Nov 19 23:19:41 2008
New Revision: 719187
URL: http://svn.apache.org/viewvc?rev=719187&view=rev
Log:
SOLR-873 -- Fix case-sensitive field names and columns
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=719187&r1=719186&r2=719187&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Wed Nov 19 23:19:41 2008
@@ -58,6 +58,8 @@
7. SOLR-864: DataImportHandler does not catch and log Errors (shalin)
+8. SOLR-873: Fix case-sensitive field names and columns (Jon Baer, shalin)
+
Documentation
----------------------
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=719187&r1=719186&r2=719187&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Wed Nov 19 23:19:41 2008
@@ -191,8 +191,6 @@
public boolean multiValued = false;
- public String nameOrColName;
-
public Map<String, String> allAttributes = new HashMap<String, String>() {
public String put(String key, String value) {
if (super.containsKey(key))
@@ -211,12 +209,6 @@
allAttributes.putAll(getAllAttributes(e));
}
- public Field(String name, boolean b) {
- name = nameOrColName = column = name;
- multiValued = b;
-
- }
-
public String getName() {
return name == null ? column : name;
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=719187&r1=719186&r2=719187&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Nov 19 23:19:41 2008
@@ -126,24 +126,12 @@
}
for (Map.Entry<String, DataConfig.Field> entry : fields.entrySet()) {
DataConfig.Field fld = entry.getValue();
- FieldType fieldType = null;
-
- try {
- fieldType = schema.getDynamicFieldType(fld.name);
- } catch (RuntimeException ex) {
- // Ignore because it may not be a dynamic field
- }
-
- if (fld.name != null) {
- if (schema.getFields().get(fld.name) == null && fieldType == null) {
- errors
- .add("The field :"
- + fld.name
- + " present in DataConfig does not have a counterpart in Solr Schema");
+ SchemaField field = schema.getFieldOrNull(fld.getName());
+ if (field == null) {
+ field = config.lowerNameVsSchemaField.get(fld.getName().toLowerCase());
+ if (field == null) {
+ errors.add("The field :" + fld.getName() + " present in DataConfig does not have a counterpart in Solr Schema");
}
- } else if (schema.getFields().get(fld.column) == null
- && fieldType == null) {
- LOG.info("Column : " + fld.column + " is not a schema field");
}
}
@@ -201,30 +189,22 @@
if (e.fields != null) {
for (DataConfig.Field f : e.fields) {
- f.nameOrColName = f.getName();
if (schema != null) {
SchemaField schemaField = schema.getFieldOrNull(f.getName());
+ if (schemaField == null) {
+ schemaField = config.lowerNameVsSchemaField.get(f.getName().toLowerCase());
+ if(schemaField != null) f.name = schemaField.getName();
+ }
if (schemaField != null) {
f.multiValued = schemaField.multiValued();
f.allAttributes.put(MULTI_VALUED, Boolean.toString(schemaField
.multiValued()));
f.allAttributes.put(TYPE, schemaField.getType().getTypeName());
- f.allAttributes.put("indexed", Boolean
- .toString(schemaField.indexed()));
+ f.allAttributes.put("indexed", Boolean.toString(schemaField.indexed()));
f.allAttributes.put("stored", Boolean.toString(schemaField.stored()));
f.allAttributes.put("defaultValue", schemaField.getDefaultValue());
} else {
-
- try {
- f.allAttributes.put(TYPE, schema.getDynamicFieldType(f.getName())
- .getTypeName());
- f.allAttributes.put(MULTI_VALUED, "true");
- f.multiValued = true;
- } catch (RuntimeException e2) {
- LOG.info("Field in data-config.xml - " + f.getName()
- + " not found in schema.xml");
- f.toWrite = false;
- }
+ f.toWrite = false;
}
}
fields.put(f.getName(), f);
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=719187&r1=719186&r2=719187&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Wed Nov 19 23:19:41 2008
@@ -379,12 +379,12 @@
sf = dataImporter.getConfig().lowerNameVsSchemaField.get(key.toLowerCase());
}
if (sf != null) {
- addFieldToDoc(entry.getValue(), key, 1.0f, sf.multiValued(), doc);
+ addFieldToDoc(entry.getValue(), sf.getName(), 1.0f, sf.multiValued(), doc);
}
//else do nothing. if we add it it may fail
} else {
if (field != null && field.toWrite) {
- addFieldToDoc(entry.getValue(), key, field.boost, field.multiValued, doc);
+ addFieldToDoc(entry.getValue(), field.getName(), field.boost, field.multiValued, doc);
}
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=719187&r1=719186&r2=719187&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java Wed Nov 19 23:19:41 2008
@@ -49,9 +49,6 @@
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
- for (DataConfig.Field field : ent.fields) {
- field.nameOrColName = field.name = field.column;
- }
MockDataSource.setIterator("select * from x", new ArrayList().iterator());
ent.dataSrc = new MockDataSource();
ent.isDocRoot = true;
@@ -80,9 +77,6 @@
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
- for (DataConfig.Field field : ent.fields) {
- field.nameOrColName = field.name = field.column;
- }
List l = new ArrayList();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
@@ -125,9 +119,6 @@
ent.isDocRoot = true;
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
- for (DataConfig.Field field : ent.fields) {
- field.nameOrColName = field.name = field.column;
- }
List l = new ArrayList();
l.add(createMap("id", 1, "desc", "one"));
l.add(createMap("id", 2, "desc", "two"));
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=719187&r1=719186&r2=719187&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java Wed Nov 19 23:19:41 2008
@@ -70,6 +70,18 @@
@Test
@SuppressWarnings("unchecked")
+ public void testSingleEntity_CaseInsensitive() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desC", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ super.runFullImport(dataConfigWithCaseInsensitiveFields);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
public void testDynamicFields() throws Exception {
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
@@ -144,4 +156,13 @@
" </entity>\n" +
" </document>\n" +
"</dataConfig>";
+
+ private final String dataConfigWithCaseInsensitiveFields = "<dataConfig>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"ID\" />\n" +
+ " <field column=\"Desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
}