You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jd...@apache.org on 2012/11/27 17:30:08 UTC
svn commit: r1414250 - in /lucene/dev/branches/branch_4x/solr: ./
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/
contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/
Author: jdyer
Date: Tue Nov 27 16:30:07 2012
New Revision: 1414250
URL: http://svn.apache.org/viewvc?rev=1414250&view=rev
Log:
SOLR-2141 / SOLR-4047 / SOLR-3842 - fix problems with VariableResolver, better test coverage
Added:
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
- copied unchanged from r1414242, lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
Modified:
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1414250&r1=1414249&r2=1414250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Tue Nov 27 16:30:07 2012
@@ -241,6 +241,9 @@ Bug Fixes
* SOLR-3960: Fixed a bug where Distributed Grouping ignored PostFilters
(Nathan Visagan, hossman)
+
+* SOLR-3842: DIH would not populate multivalued fields if the column name
+ derives from a resolved variable (James Dyer)
Other Changes
----------------------
@@ -286,6 +289,8 @@ Other Changes
both DirectSolrSpellChecker & IndexBasedSpellChecker
(Tomás Fernández Löbbe via James Dyer)
+* SOLR-2141: Better test coverage for Evaluators (James Dyer)
+
================== 4.0.0 ==================
Versions of Major Components
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java?rev=1414250&r1=1414249&r2=1414250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java Tue Nov 27 16:30:07 2012
@@ -67,6 +67,15 @@ public class DateFormatEvaluator extends
availableLocales.put(locale.toString(), locale);
}
}
+ private SimpleDateFormat getDateFormat(String pattern, Locale locale) {
+ DateFormatCacheKey dfck = new DateFormatCacheKey(locale, pattern);
+ SimpleDateFormat sdf = cache.get(dfck);
+ if(sdf == null) {
+ sdf = new SimpleDateFormat(pattern, locale);
+ cache.put(dfck, sdf);
+ }
+ return sdf;
+ }
@Override
@@ -81,15 +90,13 @@ public class DateFormatEvaluator extends
VariableWrapper wrapper = (VariableWrapper) format;
o = wrapper.resolve();
format = o.toString();
- }
+ }
Locale locale = Locale.ROOT;
if(l.size()==3) {
Object localeObj = l.get(2);
String localeStr = null;
if (localeObj instanceof VariableWrapper) {
- VariableWrapper wrapper = (VariableWrapper) localeObj;
- o = wrapper.resolve();
- localeStr = o.toString();
+ localeStr = ((VariableWrapper) localeObj).resolve().toString();
} else {
localeStr = localeObj.toString();
}
@@ -97,14 +104,9 @@ public class DateFormatEvaluator extends
if(locale==null) {
throw new DataImportHandlerException(SEVERE, "Unsupported locale: " + localeStr);
}
- }
+ }
String dateFmt = format.toString();
- DateFormatCacheKey dfck = new DateFormatCacheKey(locale, dateFmt);
- SimpleDateFormat sdf = cache.get(dfck);
- if(sdf==null) {
- sdf = new SimpleDateFormat(dateFmt, locale);
- cache.put(dfck, sdf);
- }
+ SimpleDateFormat fmt = getDateFormat(dateFmt, locale);
Date date = null;
if (o instanceof VariableWrapper) {
VariableWrapper variableWrapper = (VariableWrapper) o;
@@ -114,13 +116,7 @@ public class DateFormatEvaluator extends
} else {
String s = variableval.toString();
try {
- dfck = new DateFormatCacheKey(locale, DEFAULT_DATE_FORMAT);
- sdf = cache.get(dfck);
- if(sdf==null) {
- sdf = new SimpleDateFormat(dfck.dateFormat, dfck.locale);
- cache.put(dfck, sdf);
- }
- date = new SimpleDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
+ date = getDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
} catch (ParseException exp) {
wrapAndThrow(SEVERE, exp, "Invalid expression for date");
}
@@ -134,7 +130,7 @@ public class DateFormatEvaluator extends
wrapAndThrow(SEVERE, e, "Invalid expression for date");
}
}
- return sdf.format(date);
+ return fmt.format(date);
}
static DateMathParser getDateMathParser(Locale l) {
return new DateMathParser(TimeZone.getDefault(), l) {
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1414250&r1=1414249&r2=1414250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue Nov 27 16:30:07 2012
@@ -68,7 +68,6 @@ public class DocBuilder {
Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
- //private Map<String, Object> functionsNamespace;
private Map<String, Object> persistedProperties;
private DIHProperties propWriter;
@@ -640,11 +639,20 @@ public class DocBuilder {
if (field != null) {
for (EntityField f : field) {
String name = f.getName();
+ boolean multiValued = f.isMultiValued();
+ boolean toWrite = f.isToWrite();
if(f.isDynamicName()){
name = vr.replaceTokens(name);
+ SchemaField schemaField = dataImporter.getSchemaField(name);
+ if(schemaField == null) {
+ toWrite = false;
+ } else {
+ multiValued = schemaField.multiValued();
+ toWrite = true;
+ }
}
- if (f.isToWrite()) {
- addFieldToDoc(entry.getValue(), name, f.getBoost(), f.isMultiValued(), doc);
+ if (toWrite) {
+ addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
}
}
}
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java?rev=1414250&r1=1414249&r2=1414250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java Tue Nov 27 16:30:07 2012
@@ -63,6 +63,7 @@ public class VariableResolver {
}
public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
+ public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
public VariableResolver() {
rootNamespace = new HashMap<String,Object>();
@@ -95,7 +96,11 @@ public class VariableResolver {
r = currentLevel.get(nameParts[nameParts.length - 1]);
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
&& name.length() > FUNCTIONS_NAMESPACE.length()) {
- return resolveEvaluator(name);
+ return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
+ }
+ if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
+ && name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
+ return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
}
if (r == null) {
r = System.getProperty(name);
@@ -104,12 +109,12 @@ public class VariableResolver {
return r == null ? "" : r;
}
- private Object resolveEvaluator(String name) {
+ private Object resolveEvaluator(String namespace, String name) {
if (evaluators == null) {
return "";
}
Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
- .substring(FUNCTIONS_NAMESPACE.length()));
+ .substring(namespace.length()));
if (m.find()) {
String fname = m.group(1);
Evaluator evaluator = evaluators.get(fname);
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java?rev=1414250&r1=1414249&r2=1414250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java Tue Nov 27 16:30:07 2012
@@ -24,7 +24,7 @@ import java.text.SimpleDateFormat;
import java.util.*;
/**
- * <p> Test for EvaluatorBag </p>
+ * <p> Test for Evaluators </p>
*
*
* @since solr 1.3