You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/02/19 06:28:49 UTC
svn commit: r745734 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Thu Feb 19 05:28:48 2009
New Revision: 745734
URL: http://svn.apache.org/viewvc?rev=745734&view=rev
Log:
SOLR-996 -- Expose Context to Evaluators
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Thu Feb 19 05:28:48 2009
@@ -13,6 +13,9 @@
Upgrading from Solr 1.3
-----------------------
+Evaluator API has been changed in a non back-compatible way. Users who have developed custom Evaluators will need
+to change their code according to the new API for it to work. See SOLR-996 for details.
+
Detailed Change List
----------------------
@@ -69,6 +72,9 @@
16.SOLR-989: Expose running statistics from the Context API.
(Noble Paul, shalin)
+17.SOLR-996: Expose Context to Evaluators.
+ (Noble Paul, shalin)
+
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Thu Feb 19 05:28:48 2009
@@ -62,6 +62,7 @@
private Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
+ Map<String,Object> functionsNamespace;
public DocBuilder(DataImporter context, SolrWriter writer, DataImporter.RequestParams reqParams) {
INSTANCE.set(this);
@@ -70,6 +71,7 @@
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
requestParameters = reqParams;
verboseDebug = requestParameters.debug && requestParameters.verbose;
+ functionsNamespace = EvaluatorBag.getFunctionsNamespace(dataImporter.getConfig().functions, this);
}
public VariableResolverImpl getVariableResolver() {
@@ -80,13 +82,10 @@
DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getLastIndexTime()));
indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
indexerNamespace.put("request", requestParameters.requestParams);
- indexerNamespace.put("functions", EvaluatorBag.getFunctionsNamespace(resolver,
- dataImporter.getConfig().functions, this));
+ indexerNamespace.put("functions", functionsNamespace);
if (dataImporter.getConfig().script != null) {
- indexerNamespace
- .put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
- indexerNamespace.put(DataConfig.SCRIPT_LANG,
- dataImporter.getConfig().script.language);
+ indexerNamespace.put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
+ indexerNamespace.put(DataConfig.SCRIPT_LANG, dataImporter.getConfig().script.language);
}
resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
return resolver;
@@ -290,6 +289,7 @@
ContextImpl ctx = new ContextImpl(entity, vr, null,
pk == null ? Context.FULL_DUMP : Context.DELTA_DUMP,
session, parentCtx, this);
+ vr.context = ctx;
entityProcessor.init(ctx);
if (requestParameters.start > 0) {
@@ -517,7 +517,9 @@
Set<Map<String, Object>> deltaSet = new HashSet<Map<String, Object>>();
resolver.addNamespace(null, (Map) entity.allAttributes);
EntityProcessor entityProcessor = getEntityProcessor(entity, context.getCore());
- entityProcessor.init(new ContextImpl(entity, resolver, null, Context.FIND_DELTA, session, null, this));
+ ContextImpl context1 = new ContextImpl(entity, resolver, null, Context.FIND_DELTA, session, null, this);
+ resolver.context = context1;
+ entityProcessor.init(context1);
LOG.info("Running ModifiedRowKey() for Entity: " + entity.name);
//get the modified rows in this entity
while (true) {
@@ -560,7 +562,9 @@
//so propogate up the changes in the chain
if (parentEntity != null && parentEntity.isDocRoot) {
EntityProcessor parentEntityProcessor = getEntityProcessor(parentEntity, context.getCore());
- parentEntityProcessor.init(new ContextImpl(parentEntity, resolver, null, Context.FIND_DELTA, session, null, this));
+ ContextImpl context2 = new ContextImpl(parentEntity, resolver, null, Context.FIND_DELTA, session, null, this);
+ resolver.context = context2;
+ parentEntityProcessor.init(context2);
// identifying deleted rows with deltas
for (Map<String, Object> row : myModifiedPks)
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java Thu Feb 19 05:28:48 2009
@@ -39,9 +39,9 @@
* Return a String after processing an expression and a VariableResolver
*
* @see org.apache.solr.handler.dataimport.VariableResolver
- * @param resolver the VariableResolver instance to be used for evaluation
* @param expression string to be evaluated
+ * @param context instance
* @return the value of the given expression evaluated using the resolver
*/
- public abstract String evaluate(VariableResolver resolver, String expression);
+ public abstract String evaluate(String expression, Context context);
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java Thu Feb 19 05:28:48 2009
@@ -71,8 +71,8 @@
*/
public static Evaluator getSqlEscapingEvaluator() {
return new Evaluator() {
- public String evaluate(VariableResolver resolver, String expression) {
- Object o = resolver.resolve(expression);
+ public String evaluate(String expression, Context context) {
+ Object o = context.getVariableResolver().resolve(expression);
if (o == null)
return null;
@@ -94,10 +94,10 @@
*/
public static Evaluator getUrlEvaluator() {
return new Evaluator() {
- public String evaluate(VariableResolver resolver, String expression) {
+ public String evaluate(String expression, Context context) {
Object value = null;
try {
- value = resolver.resolve(expression);
+ value = context.getVariableResolver().resolve(expression);
if (value == null)
return null;
@@ -130,7 +130,7 @@
*/
public static Evaluator getDateFormatEvaluator() {
return new Evaluator() {
- public String evaluate(VariableResolver resolver, String expression) {
+ public String evaluate(String expression, Context context) {
CacheEntry e = getCachedData(expression);
String expr = e.key;
SimpleDateFormat fmt = e.format;
@@ -146,7 +146,7 @@
"Invalid expression for date", exp);
}
} else {
- Object o = resolver.resolve(expr);
+ Object o = context.getVariableResolver().resolve(expr);
if (o == null)
return "";
Date date = null;
@@ -193,8 +193,7 @@
};
}
- static Map<String, Object> getFunctionsNamespace(
- final VariableResolver resolver, final List<Map<String, String>> fn, DocBuilder docBuilder) {
+ static Map<String, Object> getFunctionsNamespace(final List<Map<String, String>> fn, DocBuilder docBuilder) {
final Map<String, Evaluator> evaluators = new HashMap<String, Evaluator>();
evaluators.put(DATE_FORMAT_EVALUATOR, getDateFormatEvaluator());
evaluators.put(SQL_ESCAPE_EVALUATOR, getSqlEscapingEvaluator());
@@ -222,8 +221,11 @@
Evaluator evaluator = evaluators.get(fname);
if (evaluator == null)
return null;
- return evaluator.evaluate(resolver, m.group(2));
+ VariableResolverImpl vri = VariableResolverImpl.CURRENT_VARIABLE_RESOLVER.get();
+ Context ctx = vri == null ? null : vri.context;
+ return evaluator.evaluate(m.group(2), ctx);
}
+
};
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java Thu Feb 19 05:28:48 2009
@@ -34,11 +34,21 @@
public class VariableResolverImpl extends VariableResolver {
private Map<String, Object> container = new HashMap<String, Object>();
+ /**
+ * Used for creating Evaluators
+ */
+ ContextImpl context;
+
private static final TemplateString TEMPLATE_STRING = new TemplateString();
public VariableResolverImpl() {
}
+ /**
+ * The current resolver instance
+ */
+ static final ThreadLocal<VariableResolverImpl> CURRENT_VARIABLE_RESOLVER = new ThreadLocal<VariableResolverImpl>();
+
@SuppressWarnings("unchecked")
public VariableResolverImpl addNamespace(String name, Map<String, Object> map) {
if (name != null) {
@@ -84,22 +94,27 @@
if ("".equals(name))
return null;
String[] parts = DOT_SPLIT.split(name, 0);
- Map<String, Object> namespace = container;
- for (int i = 0; i < parts.length; i++) {
- String thePart = parts[i];
- if (i == parts.length - 1) {
- return namespace.get(thePart);
- }
- Object temp = namespace.get(thePart);
- if (temp == null) {
- return namespace.get(mergeAll(parts, i));
- } else {
- if (temp instanceof Map) {
- namespace = (Map) temp;
+ CURRENT_VARIABLE_RESOLVER.set(this);
+ try {
+ Map<String, Object> namespace = container;
+ for (int i = 0; i < parts.length; i++) {
+ String thePart = parts[i];
+ if (i == parts.length - 1) {
+ return namespace.get(thePart);
+ }
+ Object temp = namespace.get(thePart);
+ if (temp == null) {
+ return namespace.get(mergeAll(parts, i));
} else {
- return null;
+ if (temp instanceof Map) {
+ namespace = (Map) temp;
+ } else {
+ return null;
+ }
}
}
+ } finally {
+ CURRENT_VARIABLE_RESOLVER.set(null);
}
return null;
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java Thu Feb 19 05:28:48 2009
@@ -23,14 +23,13 @@
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
- * <p>
- * Test for EvaluatorBag
- * </p>
+ * <p> Test for EvaluatorBag </p>
*
* @version $Id$
* @since solr 1.3
@@ -66,8 +65,7 @@
}
/**
- * Test method for
- * {@link EvaluatorBag#getSqlEscapingEvaluator()}.
+ * Test method for {@link EvaluatorBag#getSqlEscapingEvaluator()}.
*/
@Test
public void testGetSqlEscapingEvaluator() {
@@ -76,8 +74,7 @@
}
/**
- * Test method for
- * {@link EvaluatorBag#getUrlEvaluator()}.
+ * Test method for {@link EvaluatorBag#getUrlEvaluator()}.
*/
@Test
public void testGetUrlEvaluator() throws Exception {
@@ -86,32 +83,35 @@
}
/**
- * Test method for
- * {@link EvaluatorBag#getDateFormatEvaluator()}.
+ * Test method for {@link EvaluatorBag#getDateFormatEvaluator()}.
*/
@Test
@Ignore
public void testGetDateFormatEvaluator() {
Evaluator dateFormatEval = EvaluatorBag.getDateFormatEvaluator();
+ resolver.context = new ContextImpl(null, resolver, null, 0, Collections.EMPTY_MAP, null, null);
+
assertEquals(new SimpleDateFormat("yyyy-MM-dd").format(new Date()),
- dateFormatEval.evaluate(resolver, "'NOW',yyyy-MM-dd HH:mm"));
+ dateFormatEval.evaluate("'NOW',yyyy-MM-dd HH:mm", resolver.context));
Map<String, Object> map = new HashMap<String, Object>();
map.put("key", new Date());
resolver.addNamespace("A", map);
assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm").format(new Date()),
- dateFormatEval.evaluate(resolver, "A.key, yyyy-MM-dd HH:mm"));
+ dateFormatEval.evaluate("A.key, yyyy-MM-dd HH:mm", resolver.context));
}
private void runTests(Map<String, String> tests, Evaluator evaluator) {
+ ContextImpl ctx = new ContextImpl(null, resolver, null, 0, Collections.EMPTY_MAP, null, null);
+ resolver.context = ctx;
for (Map.Entry<String, String> entry : tests.entrySet()) {
Map<String, Object> values = new HashMap<String, Object>();
values.put("key", entry.getKey());
resolver.addNamespace("A", values);
String expected = (String) entry.getValue();
- String actual = evaluator.evaluate(resolver, "A.key");
+ String actual = evaluator.evaluate("A.key", ctx);
assertEquals(expected, actual);
}
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java?rev=745734&r1=745733&r2=745734&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java Thu Feb 19 05:28:48 2009
@@ -68,8 +68,10 @@
@Test
public void dateNamespaceWithValue() {
VariableResolverImpl vri = new VariableResolverImpl();
+ ContextImpl context = new ContextImpl(null,vri, null, 0,Collections.EMPTY_MAP, null,null);
+ vri.context = context;
vri.addNamespace("dataimporter.functions", EvaluatorBag
- .getFunctionsNamespace(vri, Collections.EMPTY_LIST, null));
+ .getFunctionsNamespace(Collections.EMPTY_LIST, null));
Map<String, Object> ns = new HashMap<String, Object>();
Date d = new Date();
ns.put("dt", d);
@@ -84,8 +86,10 @@
@Test
public void dateNamespaceWithExpr() {
VariableResolverImpl vri = new VariableResolverImpl();
+ ContextImpl context = new ContextImpl(null,vri, null, 0,Collections.EMPTY_MAP, null,null);
+ vri.context = context;
vri.addNamespace("dataimporter.functions", EvaluatorBag
- .getFunctionsNamespace(vri, Collections.EMPTY_LIST,null));
+ .getFunctionsNamespace(Collections.EMPTY_LIST,null));
String s = vri
.replaceTokens("${dataimporter.functions.formatDate('NOW',yyyy-MM-dd HH:mm)}");
Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm")
@@ -113,13 +117,15 @@
@Test
public void testFunctionNamespace1() {
final VariableResolverImpl resolver = new VariableResolverImpl();
+ ContextImpl context = new ContextImpl(null,resolver, null, 0,Collections.EMPTY_MAP, null,null);
+ resolver.context = context;
final List<Map<String ,String >> l = new ArrayList<Map<String, String>>();
Map<String ,String > m = new HashMap<String, String>();
m.put("name","test");
m.put("class",E.class.getName());
l.add(m);
resolver.addNamespace("dataimporter.functions", EvaluatorBag
- .getFunctionsNamespace(resolver, l,null));
+ .getFunctionsNamespace(l,null));
String s = resolver
.replaceTokens("${dataimporter.functions.formatDate('NOW',yyyy-MM-dd HH:mm)}");
Assert.assertEquals(new SimpleDateFormat("yyyy-MM-dd HH:mm")
@@ -129,7 +135,7 @@
}
public static class E extends Evaluator{
- public String evaluate(VariableResolver resolver, String expression) {
+ public String evaluate(String expression, Context context) {
return "Hello World";
}
}