You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/02/24 07:58:46 UTC
svn commit: r747291 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Tue Feb 24 06:58:46 2009
New Revision: 747291
URL: http://svn.apache.org/viewvc?rev=747291&view=rev
Log:
SOLR-1029 -- Standardize Evaluator parameter parsing and added helper functions for parsing all evaluator parameters in a standard way.
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=747291&r1=747290&r2=747291&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Tue Feb 24 06:58:46 2009
@@ -16,6 +16,10 @@
Evaluator API has been changed in a non back-compatible way. Users who have developed custom Evaluators will need
to change their code according to the new API for it to work. See SOLR-996 for details.
+The formatDate evaluator's syntax has been changed. The new syntax is formatDate(<variable>, '<format_string>').
+For example, formatDate(x.date, 'yyyy-MM-dd'). In the old syntax, the date string was written without a single-quotes.
+The old syntax has been deprecated and will be removed in 1.5, until then, using the old syntax will log a warning.
+
Detailed Change List
----------------------
@@ -149,13 +153,17 @@
Other
----------------------
-1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument.
- Refactored to load Evaluators lazily. Removed multiple document nodes in the configuration xml.
- Removed support for 'default' variables, they are automatically available as request parameters.
- (Noble Paul via shalin)
+1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument.
+ Refactored to load Evaluators lazily. Removed multiple document nodes in the configuration xml.
+ Removed support for 'default' variables, they are automatically available as request parameters.
+ (Noble Paul via shalin)
-2. SOLR-964: XPathEntityProcessor now ignores DTD validations
- (Fergus McMenemie, Noble Paul via shalin)
+2. SOLR-964: XPathEntityProcessor now ignores DTD validations
+ (Fergus McMenemie, Noble Paul via shalin)
+
+3. SOLR-1029: Standardize Evaluator parameter parsing and added helper functions for parsing all evaluator
+ parameters in a standard way.
+ (Noble Paul, shalin)
================== Release 1.3.0 20080915 ==================
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java?rev=747291&r1=747290&r2=747291&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java Tue Feb 24 06:58:46 2009
@@ -16,13 +16,16 @@
* limitations under the License.
*/
-import static org.apache.solr.handler.dataimport.DocBuilder.loadClass;
+import org.apache.solr.core.SolrCore;
import static org.apache.solr.handler.dataimport.DataConfig.CLASS;
import static org.apache.solr.handler.dataimport.DataConfig.NAME;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import static org.apache.solr.handler.dataimport.DocBuilder.loadClass;
import org.apache.solr.util.DateMathParser;
-import org.apache.solr.core.SolrCore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -31,15 +34,9 @@
import java.util.regex.Pattern;
/**
- * <p>
- * Holds definitions for evaluators provided by DataImportHandler
- * </p>
- * <p/>
- * <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
- * </p>
+ * <p> Holds definitions for evaluators provided by DataImportHandler </p> <p/> <p> Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a> for more
+ * details. </p>
* <p/>
* <b>This API is experimental and may change in the future.</b>
*
@@ -47,7 +44,8 @@
* @since solr 1.3
*/
public class EvaluatorBag {
-
+ private static final Logger LOG = LoggerFactory.getLogger(EvaluatorBag.class);
+
public static final String DATE_FORMAT_EVALUATOR = "formatDate";
public static final String URL_ENCODE_EVALUATOR = "encodeUrl";
@@ -57,139 +55,108 @@
.compile("^(\\w*?)\\((.*?)\\)$");
/**
- * <p/>
- * Returns an <code>Evaluator</code> instance meant to be used for escaping
- * values in SQL queries.
- * </p>
- * <p/>
- * It escapes the value of the given expression by replacing all occurrences
- * of single-quotes by two single-quotes and similarily for double-quotes
- * </p>
+ * <p/> Returns an <code>Evaluator</code> instance meant to be used for escaping values in SQL queries. </p> <p/> It
+ * escapes the value of the given expression by replacing all occurrences of single-quotes by two single-quotes and
+ * similarily for double-quotes </p>
*
- * @return an <code>Evaluator</code> instance capable of SQL-escaping
- * expressions.
+ * @return an <code>Evaluator</code> instance capable of SQL-escaping expressions.
*/
public static Evaluator getSqlEscapingEvaluator() {
return new Evaluator() {
public String evaluate(String expression, Context context) {
- Object o = context.getVariableResolver().resolve(expression);
-
- if (o == null)
- return null;
-
- return o.toString().replaceAll("'", "''").replaceAll("\"", "\"\"");
+ List l = parseParams(expression, context.getVariableResolver());
+ if (l.size() != 1) {
+ throw new DataImportHandlerException(SEVERE, "'escapeSql' must have at least one parameter ");
+ }
+ String s = l.get(0).toString();
+ return s.replaceAll("'", "''").replaceAll("\"", "\"\"");
}
};
}
/**
- * <p/>
- * Returns an <code>Evaluator</code> instance capable of URL-encoding
- * expressions. The expressions are evaluated using a
- * <code>VariableResolver</code>
- * </p>
+ * <p/> Returns an <code>Evaluator</code> instance capable of URL-encoding expressions. The expressions are evaluated
+ * using a <code>VariableResolver</code> </p>
*
- * @return an <code>Evaluator</code> instance capable of URL-encoding
- * expressions.
+ * @return an <code>Evaluator</code> instance capable of URL-encoding expressions.
*/
public static Evaluator getUrlEvaluator() {
return new Evaluator() {
public String evaluate(String expression, Context context) {
- Object value = null;
+ List l = parseParams(expression, context.getVariableResolver());
+ if (l.size() != 1) {
+ throw new DataImportHandlerException(SEVERE, "'encodeUrl' must have at least one parameter ");
+ }
+ String s = l.get(0).toString();
+
try {
- value = context.getVariableResolver().resolve(expression);
- if (value == null)
- return null;
-
- return URLEncoder.encode(value.toString(), "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "Unable to encode expression: " + expression + " with value: "
- + value, e);
+ return URLEncoder.encode(s.toString(), "UTF-8");
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e, "Unable to encode expression: " + expression + " with value: " + s);
+ return null;
}
}
};
}
/**
- * <p/>
- * Returns an <code>Evaluator</code> instance capable of formatting values
- * using a given date format.
- * </p>
- * <p/>
- * The value to be formatted can be a entity.field or a date expression parsed
- * with <code>DateMathParser</code> class. If the value is in single quotes,
- * then it is assumed to be a datemath expression, otherwise it resolved using
- * a <code>VariableResolver</code> instance
- * </p>
+ * <p/> Returns an <code>Evaluator</code> instance capable of formatting values using a given date format. </p> <p/>
+ * The value to be formatted can be a entity.field or a date expression parsed with <code>DateMathParser</code> class.
+ * If the value is in a String, then it is assumed to be a datemath expression, otherwise it resolved using a
+ * <code>VariableResolver</code> instance </p>
+ *
+ * @return an Evaluator instance capable of formatting values to a given date format
*
- * @return an Evaluator instance capable of formatting values to a given date
- * format
* @see DateMathParser
*/
public static Evaluator getDateFormatEvaluator() {
return new Evaluator() {
public String evaluate(String expression, Context context) {
- CacheEntry e = getCachedData(expression);
- String expr = e.key;
- SimpleDateFormat fmt = e.format;
- Matcher m = IN_SINGLE_QUOTES.matcher(expr);
- if (m.find()) {
- String datemathExpr = m.group(1);
- try {
- Date date = dateMathParser.parseMath(datemathExpr);
- return fmt.format(date);
- } catch (ParseException exp) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "Invalid expression for date", exp);
+ List l = parseParams(expression, context.getVariableResolver());
+ if (l.size() != 2) {
+ throw new DataImportHandlerException(SEVERE, "'formatDate()' must have two parameters ");
+ }
+ Object o = l.get(0);
+ Object format = l.get(1);
+ if (format instanceof VariableWrapper) {
+ VariableWrapper wrapper = (VariableWrapper) format;
+ o = wrapper.resolve();
+ if (o == null) {
+ format = wrapper.varName;
+ LOG.warn("Deprecated syntax used. The syntax of formatDate has been changed to formatDate(<var>, '<date_format_string>'). " +
+ "The old syntax will stop working in Solr 1.5");
+ } else {
+ format = o.toString();
}
- } else {
- Object o = context.getVariableResolver().resolve(expr);
- if (o == null)
- return "";
- Date date = null;
- if (o instanceof Date) {
- date = (Date) o;
+ }
+ String dateFmt = format.toString();
+ SimpleDateFormat fmt = new SimpleDateFormat(dateFmt);
+ Date date = null;
+ if (o instanceof VariableWrapper) {
+ VariableWrapper variableWrapper = (VariableWrapper) o;
+ Object variableval = variableWrapper.resolve();
+ if (variableval instanceof Date) {
+ date = (Date) variableval;
} else {
- String s = o.toString();
+ String s = variableval.toString();
try {
date = DataImporter.DATE_TIME_FORMAT.get().parse(s);
} catch (ParseException exp) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "Invalid expression for date", exp);
+ wrapAndThrow(SEVERE, exp, "Invalid expression for date");
}
}
- return fmt.format(date);
- }
- }
-
- private CacheEntry getCachedData(String str) {
- CacheEntry result = cache.get(str);
- if (result != null)
- return result;
- Matcher m = FORMAT_METHOD.matcher(str);
- String expr, pattern;
- if (m.find()) {
- expr = m.group(1).trim();
- if (IN_SINGLE_QUOTES.matcher(expr).find()) {
- expr = expr.replaceAll("NOW", "");
- }
- pattern = m.group(2).trim();
- cache.put(str, new CacheEntry(expr, new SimpleDateFormat(pattern)));
- return cache.get(str);
} else {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE, "Invalid format String : "
- + "${dataimporter.functions." + str + "}");
+ String datemathfmt = o.toString();
+ datemathfmt = datemathfmt.replaceAll("NOW", "");
+ try {
+ date = dateMathParser.parseMath(datemathfmt);
+ } catch (ParseException e) {
+ wrapAndThrow(SEVERE, e, "Invalid expression for date");
+ }
}
+ return fmt.format(date);
}
- Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
-
- Pattern FORMAT_METHOD = Pattern.compile("^(.*?),(.*?)$");
};
}
@@ -203,9 +170,7 @@
try {
evaluators.put(map.get(NAME), (Evaluator) loadClass(map.get(CLASS), core).newInstance());
} catch (Exception e) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE,
- "Unable to instantiate evaluator: " + map.get(CLASS), e);
+ wrapAndThrow(SEVERE, e, "Unable to instantiate evaluator: " + map.get(CLASS));
}
}
@@ -229,15 +194,77 @@
};
}
+ /**
+ * Parses a string of expression into separate params. The values are separated by commas. each value will be
+ * translated into one of the following:
+ * <ol>
+ * <li>If it is in single quotes the value will be translated to a String</li>
+ * <li>If is is not in quotes and is a number a it will be translated into a Double</li>
+ * <li>else it is a variable which can be resolved and it will be put in as an instance of VariableWrapper</li>
+ * </ol>
+ *
+ * @param expression the expression to be parsed
+ * @param vr the VariableResolver instance for resolving variables
+ *
+ * @return a List of objects which can either be a string, number or a variable wrapper
+ */
+ public static List parseParams(String expression, VariableResolver vr) {
+ List result = new ArrayList();
+ expression = expression.trim();
+ String[] ss = expression.split(",");
+ for (int i = 0; i < ss.length; i++) {
+ ss[i] = ss[i].trim();
+ if (ss[i].startsWith("'")) {//a string param has started
+ StringBuilder sb = new StringBuilder();
+ while (true) {
+ sb.append(ss[i]);
+ if (ss[i].endsWith("'")) break;
+ i++;
+ if (i >= ss.length)
+ throw new DataImportHandlerException(SEVERE, "invalid string at " + ss[i - 1] + " in function params: " + expression);
+ sb.append(",");
+ }
+ String s = sb.substring(1, sb.length() - 1);
+ s = s.replaceAll("\\\\'", "'");
+ result.add(s);
+ } else {
+ if (Character.isDigit(ss[i].charAt(0))) {
+ try {
+ Double doub = Double.parseDouble(ss[i]);
+ result.add(doub);
+ } catch (NumberFormatException e) {
+ if (vr.resolve(ss[i]) == null) {
+ wrapAndThrow(
+ SEVERE, e, "Invalid number :" + ss[i] +
+ "in parameters " + expression);
+ }
+ }
+ } else {
+ result.add(new VariableWrapper(ss[i], vr));
+ }
+ }
+ }
+ return result;
+ }
+
+ public static class VariableWrapper {
+ String varName;
+ VariableResolver vr;
+
+ public VariableWrapper(String s, VariableResolver vr) {
+ this.varName = s;
+ this.vr = vr;
+ }
- static class CacheEntry {
- public String key;
+ public Object resolve() {
+ return vr.resolve(varName);
+
+ }
- public SimpleDateFormat format;
+ public String toString() {
+ Object o = vr.resolve(varName);
+ return o == null ? null : o.toString();
- public CacheEntry(String key, SimpleDateFormat format) {
- this.key = key;
- this.format = format;
}
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java?rev=747291&r1=747290&r2=747291&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java Tue Feb 24 06:58:46 2009
@@ -23,10 +23,9 @@
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
+
+import junit.framework.Assert;
/**
* <p> Test for EvaluatorBag </p>
@@ -81,6 +80,19 @@
Evaluator urlEvaluator = EvaluatorBag.getUrlEvaluator();
runTests(urlTests, urlEvaluator);
}
+ @Test
+ public void parseParams() {
+ Map m = new HashMap();
+ m.put("b","B");
+ VariableResolverImpl vr = new VariableResolverImpl();
+ vr.addNamespace("a",m);
+ List l = EvaluatorBag.parseParams(" 1 , a.b, 'hello!', 'ds,o,u\'za',",vr);
+ Assert.assertEquals(new Double(1),l.get(0));
+ Assert.assertEquals("B",((EvaluatorBag.VariableWrapper)l.get(1)).resolve());
+ Assert.assertEquals("hello!",l.get(2));
+ Assert.assertEquals("ds,o,u'za",l.get(3));
+
+ }
/**
* Test method for {@link EvaluatorBag#getDateFormatEvaluator()}.
@@ -92,14 +104,14 @@
resolver.context = new ContextImpl(null, resolver, null, 0, Collections.EMPTY_MAP, null, null);
assertEquals(new SimpleDateFormat("yyyy-MM-dd").format(new Date()),
- dateFormatEval.evaluate("'NOW',yyyy-MM-dd HH:mm", resolver.context));
+ dateFormatEval.evaluate("'NOW','yyyy-MM-dd HH:mm'", resolver.context));
Map<String, Object> map = new HashMap<String, Object>();
map.put("key", new Date());
resolver.addNamespace("A", map);
assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm").format(new Date()),
- dateFormatEval.evaluate("A.key, yyyy-MM-dd HH:mm", resolver.context));
+ dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm'", resolver.context));
}
private void runTests(Map<String, String> tests, Evaluator evaluator) {
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java?rev=747291&r1=747290&r2=747291&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java Tue Feb 24 06:58:46 2009
@@ -76,11 +76,8 @@
Date d = new Date();
ns.put("dt", d);
vri.addNamespace("A", ns);
- Assert
- .assertEquals(
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(d),
- vri
- .replaceTokens("${dataimporter.functions.formatDate(A.dt,yyyy-MM-dd HH:mm:ss)}"));
+ Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(d),
+ vri.replaceTokens("${dataimporter.functions.formatDate(A.dt,'yyyy-MM-dd HH:mm:ss')}"));
}
@Test
@@ -91,7 +88,7 @@
vri.addNamespace("dataimporter.functions", EvaluatorBag
.getFunctionsNamespace(Collections.EMPTY_LIST,null));
String s = vri
- .replaceTokens("${dataimporter.functions.formatDate('NOW',yyyy-MM-dd HH:mm)}");
+ .replaceTokens("${dataimporter.functions.formatDate('NOW','yyyy-MM-dd HH:mm')}");
Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm")
.format(new Date()), s);
}
@@ -127,7 +124,7 @@
resolver.addNamespace("dataimporter.functions", EvaluatorBag
.getFunctionsNamespace(l,null));
String s = resolver
- .replaceTokens("${dataimporter.functions.formatDate('NOW',yyyy-MM-dd HH:mm)}");
+ .replaceTokens("${dataimporter.functions.formatDate('NOW','yyyy-MM-dd HH:mm')}");
Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm")
.format(new Date()), s);
Assert.assertEquals("Hello World", resolver