You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/04/29 13:23:30 UTC
svn commit: r769751 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Wed Apr 29 11:23:28 2009
New Revision: 769751
URL: http://svn.apache.org/viewvc?rev=769751&view=rev
Log:
SOLR-1127 -- Add support for field name to be templatized
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=769751&r1=769750&r2=769751&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Wed Apr 29 11:23:28 2009
@@ -136,6 +136,9 @@
for processing with transformers and child entities.
(Fergus McMenemie, Noble Paul, shalin)
+31.SOLR-1127: Add support for field name to be templatized.
+ (Noble Paul, shalin)
+
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=769751&r1=769750&r2=769751&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Wed Apr 29 11:23:28 2009
@@ -190,6 +190,9 @@
public boolean multiValued = false;
+ boolean dynamicName;
+
+
public Map<String, String> allAttributes = new HashMap<String, String>() {
public String put(String key, String value) {
if (super.containsKey(key))
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=769751&r1=769750&r2=769751&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Apr 29 11:23:28 2009
@@ -181,6 +181,10 @@
if (e.fields != null) {
for (DataConfig.Field f : e.fields) {
if (schema != null) {
+ if(f.name != null && f.name.contains("${")){
+ f.dynamicName = true;
+ continue;
+ }
SchemaField schemaField = schema.getFieldOrNull(f.getName());
if (schemaField == null) {
schemaField = config.lowerNameVsSchemaField.get(f.getName().toLowerCase());
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=769751&r1=769750&r2=769751&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Wed Apr 29 11:23:28 2009
@@ -327,7 +327,7 @@
DataConfig.Entity e = entity;
while (e.parentEntity != null) {
addFields(e.parentEntity, doc, (Map<String, Object>) vr
- .resolve(e.parentEntity.name));
+ .resolve(e.parentEntity.name), vr);
e = e.parentEntity;
}
}
@@ -335,7 +335,7 @@
Map<String, Object> arow = entityProcessor.nextRow();
if (arow == null) {
entityProcessor.destroy();
- break;
+ break;
}
// Support for start parameter in debug mode
@@ -354,7 +354,7 @@
importStatistics.rowsCount.incrementAndGet();
if (doc != null) {
handleSpecialCommands(arow, doc);
- addFields(entity, doc, arow);
+ addFields(entity, doc, arow, vr);
}
if (isRoot)
vr.removeNamespace(null);
@@ -480,7 +480,8 @@
}
@SuppressWarnings("unchecked")
- private void addFields(DataConfig.Entity entity, DocWrapper doc, Map<String, Object> arow) {
+ private void addFields(DataConfig.Entity entity, DocWrapper doc,
+ Map<String, Object> arow, VariableResolver vr) {
for (Map.Entry<String, Object> entry : arow.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
@@ -500,7 +501,11 @@
} else {
if (field != null) {
for (DataConfig.Field f : field) {
- if (f.toWrite) addFieldToDoc(entry.getValue(), f.getName(), f.boost, f.multiValued, doc);
+ String name = f.getName();
+ if(f.dynamicName){
+ name = vr.replaceTokens(name);
+ }
+ if (f.toWrite) addFieldToDoc(entry.getValue(), name, f.boost, f.multiValued, doc);
}
}
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=769751&r1=769750&r2=769751&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java Wed Apr 29 11:23:28 2009
@@ -112,6 +112,21 @@
@Test
@SuppressWarnings("unchecked")
+ public void testRequestParamsAsFieldName() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "mypk", "id", "text", "desc",
+ "dataConfig", dataConfigWithTemplatizedFieldNames);
+ h.query("/dataimport", request);
+ assertQ(req("id:101"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
public void testContext() throws Exception {
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
@@ -299,4 +314,13 @@
" </entity>\n" +
" </document>\n" +
"</dataConfig>";
+
+ private final String dataConfigWithTemplatizedFieldNames = "<dataConfig>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"mypk\" name=\"${dih.request.mypk}\" />\n" +
+ " <field column=\"text\" name=\"${dih.request.text}\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
}