You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/02/06 07:51:50 UTC
svn commit: r741435 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Fri Feb 6 06:51:50 2009
New Revision: 741435
URL: http://svn.apache.org/viewvc?rev=741435&view=rev
Log:
SOLR-1001 -- Allow variables in various attributes of RegexTransformer, HTMLStripTransformer and NumberFormatTransformer.
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=741435&r1=741434&r2=741435&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Fri Feb 6 06:51:50 2009
@@ -62,6 +62,10 @@
14.SOLR-1003: XPathEntityprocessor must allow slurping all text from a given xml node and its children.
(Noble Paul via shalin)
+15.SOLR-1001: Allow variables in various attributes of RegexTransformer, HTMLStripTransformer
+ and NumberFormatTransformer.
+ (Fergus McMenemie, Noble Paul, shalin)
+
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java?rev=741435&r1=741434&r2=741435&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java Fri Feb 6 06:51:50 2009
@@ -16,6 +16,8 @@
*/
package org.apache.solr.handler.dataimport;
+import static org.apache.solr.handler.dataimport.HTMLStripTransformer.TRUE;
+
import java.io.IOException;
import java.io.Reader;
import java.sql.Clob;
@@ -37,7 +39,7 @@
public class ClobTransformer extends Transformer {
public Object transformRow(Map<String, Object> aRow, Context context) {
for (Map<String, String> map : context.getAllEntityFields()) {
- if (!"true".equals(map.get(CLOB))) continue;
+ if (!TRUE.equals(map.get(CLOB))) continue;
String column = map.get(DataImporter.COLUMN);
String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
if (srcCol == null)
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java?rev=741435&r1=741434&r2=741435&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java Fri Feb 6 06:51:50 2009
@@ -37,10 +37,11 @@
@Override
@SuppressWarnings("unchecked")
public Object transformRow(Map<String, Object> row, Context context) {
+ VariableResolver resolver = context.getVariableResolver();
List<Map<String, String>> fields = context.getAllEntityFields();
for (Map<String, String> field : fields) {
String col = field.get(DataImporter.COLUMN);
- String splitHTML = field.get(STRIP_HTML);
+ String splitHTML = resolver.replaceTokens(field.get(STRIP_HTML));
if (!TRUE.equals(splitHTML))
continue;
Object tmpVal = row.get(col);
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java?rev=741435&r1=741434&r2=741435&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java Fri Feb 6 06:51:50 2009
@@ -51,13 +51,14 @@
@SuppressWarnings("unchecked")
public Object transformRow(Map<String, Object> row, Context context) {
+ VariableResolver resolver = context.getVariableResolver();
for (Map<String, String> fld : context.getAllEntityFields()) {
- String style = fld.get(FORMAT_STYLE);
+ String style = resolver.replaceTokens(fld.get(FORMAT_STYLE));
if (style != null) {
String column = fld.get(DataImporter.COLUMN);
String srcCol = fld.get(RegexTransformer.SRC_COL_NAME);
Locale locale = null;
- String localeStr = fld.get(LOCALE);
+ String localeStr = resolver.replaceTokens(fld.get(LOCALE));
if (srcCol == null)
srcCol = column;
if (localeStr != null) {
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java?rev=741435&r1=741434&r2=741435&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java Fri Feb 6 06:51:50 2009
@@ -45,12 +45,16 @@
@SuppressWarnings("unchecked")
public Map<String, Object> transformRow(Map<String, Object> row,
Context context) {
+ VariableResolver vr = context.getVariableResolver();
List<Map<String, String>> fields = context.getAllEntityFields();
for (Map<String, String> field : fields) {
String col = field.get(DataImporter.COLUMN);
String reStr = field.get(REGEX);
+ reStr = vr.replaceTokens(reStr);
String splitBy = field.get(SPLIT_BY);
+ splitBy = vr.replaceTokens(splitBy);
String replaceWith = field.get(REPLACE_WITH);
+ replaceWith = vr.replaceTokens(replaceWith);
if (reStr != null || splitBy != null) {
String srcColName = field.get(SRC_COL_NAME);
if (srcColName == null) {