You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/12/10 17:37:54 UTC
svn commit: r1419570 [8/14] - in /lucene/dev/branches/lucene4547: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/
dev-tools/idea/solr/contrib/dataimporthandler/ dev-tools/maven/
dev-tools/maven/lucen...
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Mon Dec 10 16:36:47 2012
@@ -31,6 +31,7 @@ import org.apache.solr.schema.SchemaFiel
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -67,17 +68,16 @@ public class DocBuilder {
Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
- private Map<String, Object> functionsNamespace;
- private Properties persistedProperties;
+ private Map<String, Object> persistedProperties;
- private DIHPropertiesWriter propWriter;
+ private DIHProperties propWriter;
private static final String PARAM_WRITER_IMPL = "writerImpl";
private static final String DEFAULT_WRITER_NAME = "SolrWriter";
private DebugLogger debugLogger;
private final RequestInfo reqParams;
@SuppressWarnings("unchecked")
- public DocBuilder(DataImporter dataImporter, SolrWriter solrWriter, DIHPropertiesWriter propWriter, RequestInfo reqParams) {
+ public DocBuilder(DataImporter dataImporter, SolrWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
INSTANCE.set(this);
this.dataImporter = dataImporter;
this.reqParams = reqParams;
@@ -85,8 +85,7 @@ public class DocBuilder {
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
verboseDebug = reqParams.isDebug() && reqParams.getDebugInfo().verbose;
persistedProperties = propWriter.readIndexerProperties();
- functionsNamespace = EvaluatorBag.getFunctionsNamespace(this.dataImporter.getConfig().getFunctions(), this, getVariableResolver());
-
+
String writerClassStr = null;
if(reqParams!=null && reqParams.getRawParams() != null) {
writerClassStr = (String) reqParams.getRawParams().get(PARAM_WRITER_IMPL);
@@ -113,30 +112,32 @@ public class DocBuilder {
return debugLogger;
}
- public VariableResolverImpl getVariableResolver() {
+ private VariableResolver getVariableResolver() {
try {
- VariableResolverImpl resolver = null;
+ VariableResolver resolver = null;
if(dataImporter != null && dataImporter.getCore() != null
&& dataImporter.getCore().getResourceLoader().getCoreProperties() != null){
- resolver = new VariableResolverImpl(dataImporter.getCore().getResourceLoader().getCoreProperties());
- } else resolver = new VariableResolverImpl();
+ resolver = new VariableResolver(dataImporter.getCore().getResourceLoader().getCoreProperties());
+ } else {
+ resolver = new VariableResolver();
+ }
+ resolver.setEvaluators(dataImporter.getEvaluators());
Map<String, Object> indexerNamespace = new HashMap<String, Object>();
- if (persistedProperties.getProperty(LAST_INDEX_TIME) != null) {
- indexerNamespace.put(LAST_INDEX_TIME, persistedProperties.getProperty(LAST_INDEX_TIME));
+ if (persistedProperties.get(LAST_INDEX_TIME) != null) {
+ indexerNamespace.put(LAST_INDEX_TIME, persistedProperties.get(LAST_INDEX_TIME));
} else {
// set epoch
- indexerNamespace.put(LAST_INDEX_TIME, DataImporter.DATE_TIME_FORMAT.get().format(EPOCH));
+ indexerNamespace.put(LAST_INDEX_TIME, EPOCH);
}
indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
indexerNamespace.put("request", reqParams.getRawParams());
- indexerNamespace.put("functions", functionsNamespace);
for (Entity entity : dataImporter.getConfig().getEntities()) {
String key = entity.getName() + "." + SolrWriter.LAST_INDEX_KEY;
- String lastIndex = persistedProperties.getProperty(key);
- if (lastIndex != null) {
+ Object lastIndex = persistedProperties.get(key);
+ if (lastIndex != null && lastIndex instanceof Date) {
indexerNamespace.put(key, lastIndex);
} else {
- indexerNamespace.put(key, DataImporter.DATE_TIME_FORMAT.get().format(EPOCH));
+ indexerNamespace.put(key, EPOCH);
}
}
resolver.addNamespace(ConfigNameConstants.IMPORTER_NS_SHORT, indexerNamespace);
@@ -149,12 +150,6 @@ public class DocBuilder {
}
}
- private Map<String,Object> getFunctionsNamespace() {
- if(functionsNamespace==null) {
-
- }
- return functionsNamespace;
- }
private void invokeEventListener(String className) {
try {
@@ -206,9 +201,8 @@ public class DocBuilder {
}
AtomicBoolean fullCleanDone = new AtomicBoolean(false);
//we must not do a delete of *:* multiple times if there are multiple root entities to be run
- Properties lastIndexTimeProps = new Properties();
- lastIndexTimeProps.setProperty(LAST_INDEX_KEY,
- DataImporter.DATE_TIME_FORMAT.get().format(dataImporter.getIndexStartTime()));
+ Map<String,Object> lastIndexTimeProps = new HashMap<String,Object>();
+ lastIndexTimeProps.put(LAST_INDEX_KEY, dataImporter.getIndexStartTime());
epwList = new ArrayList<EntityProcessorWrapper>(config.getEntities().size());
for (Entity e : config.getEntities()) {
@@ -217,8 +211,7 @@ public class DocBuilder {
for (EntityProcessorWrapper epw : epwList) {
if (entities != null && !entities.contains(epw.getEntity().getName()))
continue;
- lastIndexTimeProps.setProperty(epw.getEntity().getName() + "." + LAST_INDEX_KEY,
- DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
+ lastIndexTimeProps.put(epw.getEntity().getName() + "." + LAST_INDEX_KEY, propWriter.getCurrentTimestamp());
currentEntityProcessorWrapper = epw;
String delQuery = epw.getEntity().getAllAttributes().get("preImportDeleteQuery");
if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
@@ -243,7 +236,7 @@ public class DocBuilder {
if (stop.get()) {
// Dont commit if aborted using command=abort
- statusMessages.put("Aborted", DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
+ statusMessages.put("Aborted", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
rollback();
} else {
// Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
@@ -295,7 +288,7 @@ public class DocBuilder {
}
@SuppressWarnings("unchecked")
- private void finish(Properties lastIndexTimeProps) {
+ private void finish(Map<String,Object> lastIndexTimeProps) {
LOG.info("Import completed successfully");
statusMessages.put("", "Indexing completed. Added/Updated: "
+ importStatistics.docCount + " documents. Deleted "
@@ -329,7 +322,7 @@ public class DocBuilder {
@SuppressWarnings("unchecked")
private void doDelta() {
addStatusMessage("Delta Dump started");
- VariableResolverImpl resolver = getVariableResolver();
+ VariableResolver resolver = getVariableResolver();
if (config.getDeleteQuery() != null) {
writer.deleteByQuery(config.getDeleteQuery());
@@ -352,7 +345,7 @@ public class DocBuilder {
writer.setDeltaKeys(allPks);
statusMessages.put("Total Changed Documents", allPks.size());
- VariableResolverImpl vri = getVariableResolver();
+ VariableResolver vri = getVariableResolver();
Iterator<Map<String, Object>> pkIter = allPks.iterator();
while (pkIter.hasNext()) {
Map<String, Object> map = pkIter.next();
@@ -392,7 +385,7 @@ public class DocBuilder {
@SuppressWarnings("unchecked")
public void addStatusMessage(String msg) {
- statusMessages.put(msg, DataImporter.DATE_TIME_FORMAT.get().format(new Date()));
+ statusMessages.put(msg, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
}
private void resetEntity(EntityProcessorWrapper epw) {
@@ -403,7 +396,7 @@ public class DocBuilder {
}
- private void buildDocument(VariableResolverImpl vr, DocWrapper doc,
+ private void buildDocument(VariableResolver vr, DocWrapper doc,
Map<String,Object> pk, EntityProcessorWrapper epw, boolean isRoot,
ContextImpl parentCtx) {
List<EntityProcessorWrapper> entitiesToDestroy = new ArrayList<EntityProcessorWrapper>();
@@ -420,7 +413,7 @@ public class DocBuilder {
}
@SuppressWarnings("unchecked")
- private void buildDocument(VariableResolverImpl vr, DocWrapper doc,
+ private void buildDocument(VariableResolver vr, DocWrapper doc,
Map<String, Object> pk, EntityProcessorWrapper epw, boolean isRoot,
ContextImpl parentCtx, List<EntityProcessorWrapper> entitiesToDestroy) {
@@ -646,11 +639,20 @@ public class DocBuilder {
if (field != null) {
for (EntityField f : field) {
String name = f.getName();
+ boolean multiValued = f.isMultiValued();
+ boolean toWrite = f.isToWrite();
if(f.isDynamicName()){
name = vr.replaceTokens(name);
+ SchemaField schemaField = dataImporter.getSchemaField(name);
+ if(schemaField == null) {
+ toWrite = false;
+ } else {
+ multiValued = schemaField.multiValued();
+ toWrite = true;
+ }
}
- if (f.isToWrite()) {
- addFieldToDoc(entry.getValue(), name, f.getBoost(), f.isMultiValued(), doc);
+ if (toWrite) {
+ addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
}
}
}
@@ -707,24 +709,32 @@ public class DocBuilder {
}
private String findMatchingPkColumn(String pk, Map<String, Object> row) {
- if (row.containsKey(pk))
- throw new IllegalArgumentException(
- String.format("deltaQuery returned a row with null for primary key %s", pk));
+ if (row.containsKey(pk)) {
+ throw new IllegalArgumentException(String.format(Locale.ROOT,
+ "deltaQuery returned a row with null for primary key %s", pk));
+ }
String resolvedPk = null;
for (String columnName : row.keySet()) {
if (columnName.endsWith("." + pk) || pk.endsWith("." + columnName)) {
if (resolvedPk != null)
throw new IllegalArgumentException(
- String.format(
+ String.format(Locale.ROOT,
"deltaQuery has more than one column (%s and %s) that might resolve to declared primary key pk='%s'",
resolvedPk, columnName, pk));
resolvedPk = columnName;
}
}
- if (resolvedPk == null)
+ if (resolvedPk == null) {
throw new IllegalArgumentException(
- String.format("deltaQuery has no column to resolve to declared primary key pk='%s'", pk));
- LOG.info(String.format("Resolving deltaQuery column '%s' to match entity's declared pk '%s'", resolvedPk, pk));
+ String
+ .format(
+ Locale.ROOT,
+ "deltaQuery has no column to resolve to declared primary key pk='%s'",
+ pk));
+ }
+ LOG.info(String.format(Locale.ROOT,
+ "Resolving deltaQuery column '%s' to match entity's declared pk '%s'",
+ resolvedPk, pk));
return resolvedPk;
}
@@ -736,7 +746,7 @@ public class DocBuilder {
* @return an iterator to the list of keys for which Solr documents should be updated.
*/
@SuppressWarnings("unchecked")
- public Set<Map<String, Object>> collectDelta(EntityProcessorWrapper epw, VariableResolverImpl resolver,
+ public Set<Map<String, Object>> collectDelta(EntityProcessorWrapper epw, VariableResolver resolver,
Set<Map<String, Object>> deletedRows) {
//someone called abort
if (stop.get())
@@ -818,14 +828,16 @@ public class DocBuilder {
// identifying deleted rows with deltas
for (Map<String, Object> row : myModifiedPks) {
- getModifiedParentRows(resolver.addNamespace(epw.getEntity().getName(), row), epw.getEntity().getName(), epw, parentKeyList);
+ resolver.addNamespace(epw.getEntity().getName(), row);
+ getModifiedParentRows(resolver, epw.getEntity().getName(), epw, parentKeyList);
// check for abort
if (stop.get())
return new HashSet();
}
// running the same for deletedrows
for (Map<String, Object> row : deletedSet) {
- getModifiedParentRows(resolver.addNamespace(epw.getEntity().getName(), row), epw.getEntity().getName(), epw, parentKeyList);
+ resolver.addNamespace(epw.getEntity().getName(), row);
+ getModifiedParentRows(resolver, epw.getEntity().getName(), epw, parentKeyList);
// check for abort
if (stop.get())
return new HashSet();
@@ -840,7 +852,7 @@ public class DocBuilder {
myModifiedPks : new HashSet<Map<String, Object>>(parentKeyList);
}
- private void getModifiedParentRows(VariableResolverImpl resolver,
+ private void getModifiedParentRows(VariableResolver resolver,
String entity, EntityProcessor entityProcessor,
Set<Map<String, Object>> parentKeyList) {
try {
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java Mon Dec 10 16:36:47 2012
@@ -48,7 +48,7 @@ public class EntityProcessorWrapper exte
private boolean initalized;
private String onError;
private Context context;
- private VariableResolverImpl resolver;
+ private VariableResolver resolver;
private String entityName;
protected List<Transformer> transformers;
@@ -65,7 +65,7 @@ public class EntityProcessorWrapper exte
public void init(Context context) {
rowcache = null;
this.context = context;
- resolver = (VariableResolverImpl) context.getVariableResolver();
+ resolver = (VariableResolver) context.getVariableResolver();
if (entityName == null) {
onError = resolver.replaceTokens(context.getEntityAttribute(ON_ERROR));
if (onError == null) onError = ABORT;
@@ -171,7 +171,7 @@ public class EntityProcessorWrapper exte
Map<String, Object> transformedRow = row;
List<Map<String, Object>> rows = null;
boolean stopTransform = checkStopTransform(row);
- VariableResolverImpl resolver = (VariableResolverImpl) context.getVariableResolver();
+ VariableResolver resolver = (VariableResolver) context.getVariableResolver();
for (Transformer t : transformers) {
if (stopTransform) break;
try {
@@ -288,8 +288,8 @@ public class EntityProcessorWrapper exte
delegate.destroy();
}
- public VariableResolverImpl getVariableResolver() {
- return (VariableResolverImpl) context.getVariableResolver();
+ public VariableResolver getVariableResolver() {
+ return (VariableResolver) context.getVariableResolver();
}
public Context getContext() {
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Evaluator.java Mon Dec 10 16:36:47 2012
@@ -16,6 +16,18 @@
*/
package org.apache.solr.handler.dataimport;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
+import java.util.regex.Pattern;
+
+import org.apache.solr.util.DateMathParser;
+
/**
* <p>
* Pluggable functions for resolving variables
@@ -43,4 +55,88 @@ public abstract class Evaluator {
* @return the value of the given expression evaluated using the resolver
*/
public abstract String evaluate(String expression, Context context);
+
+ /**
+ * Parses a string of expression into separate params. The values are separated by commas. each value will be
+ * translated into one of the following:
+ * <ol>
+ * <li>If it is in single quotes the value will be translated to a String</li>
+ * <li>If is is not in quotes and is a number a it will be translated into a Double</li>
+ * <li>else it is a variable which can be resolved and it will be put in as an instance of VariableWrapper</li>
+ * </ol>
+ *
+ * @param expression the expression to be parsed
+ * @param vr the VariableResolver instance for resolving variables
+ *
+ * @return a List of objects which can either be a string, number or a variable wrapper
+ */
+ List<Object> parseParams(String expression, VariableResolver vr) {
+ List<Object> result = new ArrayList<Object>();
+ expression = expression.trim();
+ String[] ss = expression.split(",");
+ for (int i = 0; i < ss.length; i++) {
+ ss[i] = ss[i].trim();
+ if (ss[i].startsWith("'")) {//a string param has started
+ StringBuilder sb = new StringBuilder();
+ while (true) {
+ sb.append(ss[i]);
+ if (ss[i].endsWith("'")) break;
+ i++;
+ if (i >= ss.length)
+ throw new DataImportHandlerException(SEVERE, "invalid string at " + ss[i - 1] + " in function params: " + expression);
+ sb.append(",");
+ }
+ String s = sb.substring(1, sb.length() - 1);
+ s = s.replaceAll("\\\\'", "'");
+ result.add(s);
+ } else {
+ if (Character.isDigit(ss[i].charAt(0))) {
+ try {
+ Double doub = Double.parseDouble(ss[i]);
+ result.add(doub);
+ } catch (NumberFormatException e) {
+ if (vr.resolve(ss[i]) == null) {
+ wrapAndThrow(
+ SEVERE, e, "Invalid number :" + ss[i] +
+ "in parameters " + expression);
+ }
+ }
+ } else {
+ result.add(new VariableWrapper(ss[i], vr));
+ }
+ }
+ }
+ return result;
+ }
+
+ static class VariableWrapper {
+ String varName;
+ VariableResolver vr;
+
+ public VariableWrapper(String s, VariableResolver vr) {
+ this.varName = s;
+ this.vr = vr;
+ }
+
+ public Object resolve() {
+ return vr.resolve(varName);
+
+ }
+
+ @Override
+ public String toString() {
+ Object o = vr.resolve(varName);
+ return o == null ? null : o.toString();
+ }
+ }
+
+ static Pattern IN_SINGLE_QUOTES = Pattern.compile("^'(.*?)'$");
+
+ public static final String DATE_FORMAT_EVALUATOR = "formatDate";
+
+ public static final String URL_ENCODE_EVALUATOR = "encodeUrl";
+
+ public static final String ESCAPE_SOLR_QUERY_CHARS = "escapeQueryChars";
+
+ public static final String SQL_ESCAPE_EVALUATOR = "escapeSql";
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java Mon Dec 10 16:36:47 2012
@@ -106,7 +106,7 @@ public class FieldReaderDataSource exten
private Reader getReader(Blob blob)
throws SQLException, UnsupportedEncodingException {
if (encoding == null) {
- return (new InputStreamReader(blob.getBinaryStream()));
+ return (new InputStreamReader(blob.getBinaryStream(), "UTF-8"));
} else {
return (new InputStreamReader(blob.getBinaryStream(), encoding));
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java Mon Dec 10 16:36:47 2012
@@ -125,7 +125,7 @@ public class FileDataSource extends Data
protected Reader openStream(File file) throws FileNotFoundException,
UnsupportedEncodingException {
if (encoding == null) {
- return new InputStreamReader(new FileInputStream(file));
+ return new InputStreamReader(new FileInputStream(file), "UTF-8");
} else {
return new InputStreamReader(new FileInputStream(file), encoding);
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java Mon Dec 10 16:36:47 2012
@@ -19,6 +19,7 @@ package org.apache.solr.handler.dataimpo
import java.io.File;
import java.io.FilenameFilter;
import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -151,19 +152,19 @@ public class FileListEntityProcessor ext
} else {
dateStr = context.replaceTokens(dateStr);
}
- m = EvaluatorBag.IN_SINGLE_QUOTES.matcher(dateStr);
+ m = Evaluator.IN_SINGLE_QUOTES.matcher(dateStr);
if (m.find()) {
String expr = null;
expr = m.group(1).replaceAll("NOW", "");
try {
- return EvaluatorBag.dateMathParser.parseMath(expr);
+ return DateFormatEvaluator.getDateMathParser(Locale.ROOT).parseMath(expr);
} catch (ParseException exp) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Invalid expression for date", exp);
}
}
try {
- return DataImporter.DATE_TIME_FORMAT.get().parse(dateStr);
+ return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).parse(dateStr);
} catch (ParseException exp) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Invalid expression for date", exp);
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java Mon Dec 10 16:36:47 2012
@@ -397,6 +397,12 @@ public class JdbcDataSource extends
private void closeConnection() {
try {
if (conn != null) {
+ try {
+ //SOLR-2045
+ conn.commit();
+ } catch(Exception ex) {
+ //ignore.
+ }
conn.close();
}
} catch (Exception e) {
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java Mon Dec 10 16:36:47 2012
@@ -68,7 +68,7 @@ public class NumberFormatTransformer ext
throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Invalid Locale specified for field: " + fld);
}
} else {
- locale = Locale.getDefault();
+ locale = Locale.ROOT;
}
Object val = row.get(srcCol);
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java Mon Dec 10 16:36:47 2012
@@ -1,4 +1,5 @@
package org.apache.solr.handler.dataimport;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,108 +17,213 @@ package org.apache.solr.handler.dataimpo
* limitations under the License.
*/
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
import java.util.Properties;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-public class SimplePropertiesWriter implements DIHPropertiesWriter {
- private static final Logger log = LoggerFactory.getLogger(SimplePropertiesWriter.class);
-
- static final String IMPORTER_PROPERTIES = "dataimport.properties";
-
+/**
+ * <p>
+ * Writes properties using {@link Properties#store} .
+ * The special property "last_index_time" is converted to a formatted date.
+ * Users can configure the location, filename, locale and date format to use.
+ * </p>
+ */
+public class SimplePropertiesWriter extends DIHProperties {
+ private static final Logger log = LoggerFactory
+ .getLogger(SimplePropertiesWriter.class);
+
static final String LAST_INDEX_KEY = "last_index_time";
-
- private String persistFilename = IMPORTER_PROPERTIES;
-
- private String configDir = null;
-
-
-
- public void init(DataImporter dataImporter) {
+
+ protected String filename = null;
+
+ protected String configDir = null;
+
+ protected Locale locale = null;
+
+ protected SimpleDateFormat dateFormat = null;
+
+ /**
+ * The locale to use when writing the properties file. Default is {@link Locale#ROOT}
+ */
+ public static final String LOCALE = "locale";
+ /**
+ * The date format to use when writing values for "last_index_time" to the properties file.
+ * See {@link SimpleDateFormat} for patterns. Default is yyyy-MM-dd HH:mm:ss .
+ */
+ public static final String DATE_FORMAT = "dateFormat";
+ /**
+ * The directory to save the properties file in. Default is the current core's "config" directory.
+ */
+ public static final String DIRECTORY = "directory";
+ /**
+ * The filename to save the properties file to. Default is this Handler's name from solrconfig.xml.
+ */
+ public static final String FILENAME = "filename";
+
+ @Override
+ public void init(DataImporter dataImporter, Map<String, String> params) {
+ if(params.get(FILENAME) != null) {
+ filename = params.get(FILENAME);
+ } else if(dataImporter.getHandlerName()!=null) {
+ filename = dataImporter.getHandlerName() + ".properties";
+ } else {
+ filename = "dataimport.properties";
+ }
+ if(params.get(DIRECTORY) != null) {
+ configDir = params.get(DIRECTORY);
+ } else {
SolrCore core = dataImporter.getCore();
- String configDir = core ==null ? ".": core.getResourceLoader().getConfigDir();
- String persistFileName = dataImporter.getHandlerName();
-
- this.configDir = configDir;
- if(persistFileName != null){
- persistFilename = persistFileName + ".properties";
- }
+ configDir = (core == null ? "." : core.getResourceLoader().getConfigDir());
}
-
-
-
-
+ if(params.get(LOCALE) != null) {
+ String localeStr = params.get(LOCALE);
+ for (Locale l : Locale.getAvailableLocales()) {
+ if(localeStr.equals(l.getDisplayName())) {
+ locale = l;
+ break;
+ }
+ }
+ if(locale==null) {
+ throw new DataImportHandlerException(SEVERE, "Unsupported locale for PropertWriter: " + localeStr);
+ }
+ } else {
+ locale = Locale.ROOT;
+ }
+ if(params.get(DATE_FORMAT) != null) {
+ dateFormat = new SimpleDateFormat(params.get(DATE_FORMAT), locale);
+ } else {
+ dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", locale);
+ }
+ }
+
private File getPersistFile() {
String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator))
- filePath += File.separator;
- filePath += persistFilename;
+ if (configDir != null && !configDir.endsWith(File.separator)) filePath += File.separator;
+ filePath += filename;
return new File(filePath);
}
-
- public boolean isWritable() {
- File persistFile = getPersistFile();
- return persistFile.exists() ? persistFile.canWrite() : persistFile.getParentFile().canWrite();
-
+ @Override
+ public boolean isWritable() {
+ File persistFile = getPersistFile();
+ return persistFile.exists() ? persistFile.canWrite() : persistFile
+ .getParentFile().canWrite();
+
+ }
+
+ protected String convertDateToString(Date d) {
+ return dateFormat.format(d);
+ }
+ protected Date convertStringToDate(String s) {
+ try {
+ return dateFormat.parse(s);
+ } catch (ParseException e) {
+ throw new DataImportHandlerException(SEVERE, "Value for "
+ + LAST_INDEX_KEY + " is invalid for date format "
+ + dateFormat.toLocalizedPattern() + " : " + s);
}
-
- @Override
- public void persist(Properties p) {
- OutputStream propOutput = null;
-
- Properties props = readIndexerProperties();
-
+ }
+ /**
+ * {@link DocBuilder} sends the date as an Object because
+ * this class knows how to convert it to a String
+ */
+ protected Properties mapToProperties(Map<String,Object> propObjs) {
+ Properties p = new Properties();
+ for(Map.Entry<String,Object> entry : propObjs.entrySet()) {
+ String key = entry.getKey();
+ String val = null;
+ String lastKeyPart = key;
+ int lastDotPos = key.lastIndexOf('.');
+ if(lastDotPos!=-1 && key.length() > lastDotPos+1) {
+ lastKeyPart = key.substring(lastDotPos + 1);
+ }
+ if(LAST_INDEX_KEY.equals(lastKeyPart) && entry.getValue() instanceof Date) {
+ val = convertDateToString((Date) entry.getValue());
+ } else {
+ val = entry.getValue().toString();
+ }
+ p.put(key, val);
+ }
+ return p;
+ }
+ /**
+ * We'll send everything back as Strings as this class has
+ * already converted them.
+ */
+ protected Map<String,Object> propertiesToMap(Properties p) {
+ Map<String,Object> theMap = new HashMap<String,Object>();
+ for(Map.Entry<Object,Object> entry : p.entrySet()) {
+ String key = entry.getKey().toString();
+ Object val = entry.getValue().toString();
+ theMap.put(key, val);
+ }
+ return theMap;
+ }
+
+ @Override
+ public void persist(Map<String, Object> propObjs) {
+ OutputStream propOutput = null;
+ Properties existingProps = mapToProperties(readIndexerProperties());
+ Properties newProps = mapToProperties(propObjs);
try {
- props.putAll(p);
+ existingProps.putAll(newProps);
String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator))
+ if (configDir != null && !configDir.endsWith(File.separator)) {
filePath += File.separator;
- filePath += persistFilename;
+ }
+ filePath += filename;
propOutput = new FileOutputStream(filePath);
- props.store(propOutput, null);
- log.info("Wrote last indexed time to " + persistFilename);
+ existingProps.store(propOutput, null);
+ log.info("Wrote last indexed time to " + filename);
} catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Unable to persist Index Start Time", e);
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to persist Index Start Time", e);
} finally {
try {
- if (propOutput != null)
- propOutput.close();
+ if (propOutput != null) propOutput.close();
} catch (IOException e) {
propOutput = null;
}
}
}
-
+
@Override
- public Properties readIndexerProperties() {
+ public Map<String, Object> readIndexerProperties() {
Properties props = new Properties();
- InputStream propInput = null;
-
+ InputStream propInput = null;
try {
- propInput = new FileInputStream(configDir + persistFilename);
+ String filePath = configDir;
+ if (configDir != null && !configDir.endsWith(File.separator)) {
+ filePath += File.separator;
+ }
+ filePath += filename;
+ propInput = new FileInputStream(filePath);
props.load(propInput);
- log.info("Read " + persistFilename);
+ log.info("Read " + filename);
} catch (Exception e) {
- log.warn("Unable to read: " + persistFilename);
+ log.warn("Unable to read: " + filename);
} finally {
try {
- if (propInput != null)
- propInput.close();
+ if (propInput != null) propInput.close();
} catch (IOException e) {
propInput = null;
}
- }
-
- return props;
+ }
+ return propertiesToMap(props);
}
-
+
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/TemplateTransformer.java Mon Dec 10 16:36:47 2012
@@ -55,7 +55,7 @@ public class TemplateTransformer extends
@SuppressWarnings("unchecked")
public Object transformRow(Map<String, Object> row, Context context) {
- VariableResolverImpl resolver = (VariableResolverImpl) context
+ VariableResolver resolver = (VariableResolver) context
.getVariableResolver();
// Add current row to the copy of resolver map
// for (Map.Entry<String, Object> entry : row.entrySet())
@@ -69,7 +69,11 @@ public class TemplateTransformer extends
// Verify if all variables can be resolved or not
boolean resolvable = true;
- List<String> variables = getVars(expr);
+ List<String> variables = this.templateVsVars.get(expr);
+ if(variables == null){
+ variables = resolver.getVariables(expr);
+ this.templateVsVars.put(expr, variables);
+ }
for (String v : variables) {
if (resolver.resolve(v) == null) {
LOG.warn("Unable to resolve variable: " + v
@@ -91,15 +95,5 @@ public class TemplateTransformer extends
return row;
}
-
- private List<String> getVars(String expr) {
- List<String> result = this.templateVsVars.get(expr);
- if(result == null){
- result = TemplateString.getVariables(expr);
- this.templateVsVars.put(expr, result);
- }
- return result;
- }
-
public static final String TEMPLATE = "template";
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java Mon Dec 10 16:36:47 2012
@@ -16,37 +16,210 @@
*/
package org.apache.solr.handler.dataimport;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.WeakHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/**
* <p>
- * This class is more or less like a Map. But has more intelligence to resolve
- * namespaces. Namespaces are delimited with '.' (period)
+ * A set of nested maps that can resolve variables by namespaces. Variables are
+ * enclosed with a dollar sign then an opening curly brace, ending with a
+ * closing curly brace. Namespaces are delimited with '.' (period).
+ * </p>
+ * <p>
+ * This class also has special logic to resolve evaluator calls by recognizing
+ * the reserved function namespace: dataimporter.functions.xxx
* </p>
- * <p/>
* <p>
- * Refer to <a
- * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
- * for more details.
+ * This class caches strings that have already been resolved from the current
+ * dih import.
* </p>
- * <p/>
* <b>This API is experimental and may change in the future.</b>
- *
- *
+ *
+ *
* @since solr 1.3
*/
-public abstract class VariableResolver {
-
+public class VariableResolver {
+
+ private static final Pattern DOT_PATTERN = Pattern.compile("[.]");
+ private static final Pattern PLACEHOLDER_PATTERN = Pattern
+ .compile("[$][{](.*?)[}]");
+ private static final Pattern EVALUATOR_FORMAT_PATTERN = Pattern
+ .compile("^(\\w*?)\\((.*?)\\)$");
+ private Map<String,Object> rootNamespace;
+ private Map<String,Evaluator> evaluators;
+ private Map<String,Resolved> cache = new WeakHashMap<String,Resolved>();
+
+ class Resolved {
+ List<Integer> startIndexes = new ArrayList<Integer>(2);
+ List<Integer> endOffsets = new ArrayList<Integer>(2);
+ List<String> variables = new ArrayList<String>(2);
+ }
+
+ public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
+ public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
+
+ public VariableResolver() {
+ rootNamespace = new HashMap<String,Object>();
+ }
+
+ public VariableResolver(Properties defaults) {
+ rootNamespace = new HashMap<String,Object>();
+ for (Map.Entry<Object,Object> entry : defaults.entrySet()) {
+ rootNamespace.put(entry.getKey().toString(), entry.getValue());
+ }
+ }
+
+ public VariableResolver(Map<String,Object> defaults) {
+ rootNamespace = new HashMap<String,Object>(defaults);
+ }
+
/**
* Resolves a given value with a name
- *
- * @param name the String to be resolved
+ *
+ * @param name
+ * the String to be resolved
* @return an Object which is the result of evaluation of given name
*/
- public abstract Object resolve(String name);
-
+ public Object resolve(String name) {
+ Object r = null;
+ if (name != null) {
+ String[] nameParts = DOT_PATTERN.split(name);
+ Map<String,Object> currentLevel = currentLevelMap(nameParts,
+ rootNamespace, false);
+ r = currentLevel.get(nameParts[nameParts.length - 1]);
+ if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
+ && name.length() > FUNCTIONS_NAMESPACE.length()) {
+ return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
+ }
+ if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
+ && name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
+ return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
+ }
+ if (r == null) {
+ r = System.getProperty(name);
+ }
+ }
+ return r == null ? "" : r;
+ }
+
+ private Object resolveEvaluator(String namespace, String name) {
+ if (evaluators == null) {
+ return "";
+ }
+ Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
+ .substring(namespace.length()));
+ if (m.find()) {
+ String fname = m.group(1);
+ Evaluator evaluator = evaluators.get(fname);
+ if (evaluator == null) return "";
+ ContextImpl ctx = new ContextImpl(null, this, null, null, null, null,
+ null);
+ String g2 = m.group(2);
+ return evaluator.evaluate(g2, ctx);
+ } else {
+ return "";
+ }
+ }
+
/**
* Given a String with place holders, replace them with the value tokens.
- *
+ *
* @return the string with the placeholders replaced with their values
*/
- public abstract String replaceTokens(String template);
+ public String replaceTokens(String template) {
+ if (template == null) {
+ return null;
+ }
+ Resolved r = getResolved(template);
+ if (r.startIndexes != null) {
+ StringBuilder sb = new StringBuilder(template);
+ for (int i = r.startIndexes.size() - 1; i >= 0; i--) {
+ String replacement = resolve(r.variables.get(i)).toString();
+ sb.replace(r.startIndexes.get(i), r.endOffsets.get(i), replacement);
+ }
+ return sb.toString();
+ } else {
+ return template;
+ }
+ }
+
+ private Resolved getResolved(String template) {
+ Resolved r = cache.get(template);
+ if (r == null) {
+ r = new Resolved();
+ Matcher m = PLACEHOLDER_PATTERN.matcher(template);
+ while (m.find()) {
+ String variable = m.group(1);
+ r.startIndexes.add(m.start(0));
+ r.endOffsets.add(m.end(0));
+ r.variables.add(variable);
+ }
+ cache.put(template, r);
+ }
+ return r;
+ }
+ /**
+ * Get a list of variables embedded in the template string.
+ */
+ public List<String> getVariables(String template) {
+ Resolved r = getResolved(template);
+ if (r == null) {
+ return Collections.emptyList();
+ }
+ return new ArrayList<String>(r.variables);
+ }
+
+ public void addNamespace(String name, Map<String,Object> newMap) {
+ if (newMap != null) {
+ if (name != null) {
+ String[] nameParts = DOT_PATTERN.split(name);
+ Map<String,Object> nameResolveLevel = currentLevelMap(nameParts,
+ rootNamespace, false);
+ nameResolveLevel.put(nameParts[nameParts.length - 1], newMap);
+ } else {
+ for (Map.Entry<String,Object> entry : newMap.entrySet()) {
+ String[] keyParts = DOT_PATTERN.split(entry.getKey());
+ Map<String,Object> currentLevel = rootNamespace;
+ currentLevel = currentLevelMap(keyParts, currentLevel, false);
+ currentLevel.put(keyParts[keyParts.length - 1], entry.getValue());
+ }
+ }
+ }
+ }
+
+ private Map<String,Object> currentLevelMap(String[] keyParts,
+ Map<String,Object> currentLevel, boolean includeLastLevel) {
+ int j = includeLastLevel ? keyParts.length : keyParts.length - 1;
+ for (int i = 0; i < j; i++) {
+ Object o = currentLevel.get(keyParts[i]);
+ if (o == null) {
+ Map<String,Object> nextLevel = new HashMap<String,Object>();
+ currentLevel.put(keyParts[i], nextLevel);
+ currentLevel = nextLevel;
+ } else if (o instanceof Map<?,?>) {
+ @SuppressWarnings("unchecked")
+ Map<String,Object> nextLevel = (Map<String,Object>) o;
+ currentLevel = nextLevel;
+ } else {
+ throw new AssertionError(
+ "Non-leaf nodes should be of type java.util.Map");
+ }
+ }
+ return currentLevel;
+ }
+
+ public void removeNamespace(String name) {
+ rootNamespace.remove(name);
+ }
+
+ public void setEvaluators(Map<String,Evaluator> evaluators) {
+ this.evaluators = evaluators;
+ }
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java Mon Dec 10 16:36:47 2012
@@ -91,14 +91,14 @@ public class XPathEntityProcessor extend
public void init(Context context) {
super.init(context);
if (xpathReader == null)
- initXpathReader();
+ initXpathReader(context.getVariableResolver());
pk = context.getEntityAttribute("pk");
dataSource = context.getDataSource();
rowIterator = null;
}
- private void initXpathReader() {
+ private void initXpathReader(VariableResolver resolver) {
useSolrAddXml = Boolean.parseBoolean(context
.getEntityAttribute(USE_SOLR_ADD_SCHEMA));
streamRows = Boolean.parseBoolean(context
@@ -175,7 +175,7 @@ public class XPathEntityProcessor extend
}
}
String url = context.getEntityAttribute(URL);
- List<String> l = url == null ? Collections.EMPTY_LIST : TemplateString.getVariables(url);
+ List<String> l = url == null ? Collections.EMPTY_LIST : resolver.getVariables(url);
for (String s : l) {
if (s.startsWith(entityName + ".")) {
if (placeHolderVariables == null)
@@ -259,7 +259,7 @@ public class XPathEntityProcessor extend
Object val = context.getSessionAttribute(name, Context.SCOPE_ENTITY);
if (val != null) namespace.put(name, val);
}
- ((VariableResolverImpl)context.getVariableResolver()).addNamespace(entityName, namespace);
+ ((VariableResolver)context.getVariableResolver()).addNamespace(entityName, namespace);
}
private void addCommonFields(Map<String, Object> r) {
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java Mon Dec 10 16:36:47 2012
@@ -18,6 +18,7 @@ package org.apache.solr.handler.dataimpo
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.util.Map;
import java.util.Properties;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -25,7 +26,13 @@ import org.apache.zookeeper.KeeperExcept
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class ZKPropertiesWriter implements DIHPropertiesWriter {
+/**
+ * <p>
+ * A SolrCloud-friendly extension of {@link SimplePropertiesWriter}.
+ * This implementation ignores the "directory" parameter, saving
+ * the properties file under /configs/[solrcloud collection name]/
+ */
+public class ZKPropertiesWriter extends SimplePropertiesWriter {
private static final Logger log = LoggerFactory
.getLogger(ZKPropertiesWriter.class);
@@ -34,16 +41,11 @@ public class ZKPropertiesWriter implemen
private SolrZkClient zkClient;
@Override
- public void init(DataImporter dataImporter) {
+ public void init(DataImporter dataImporter, Map<String, String> params) {
+ super.init(dataImporter, params);
String collection = dataImporter.getCore().getCoreDescriptor()
.getCloudDescriptor().getCollectionName();
- String persistFilename;
- if(dataImporter.getHandlerName() != null){
- persistFilename = dataImporter.getHandlerName() + ".properties";
- } else {
- persistFilename = SimplePropertiesWriter.IMPORTER_PROPERTIES;
- }
- path = "/configs/" + collection + "/" + persistFilename;
+ path = "/configs/" + collection + "/" + filename;
zkClient = dataImporter.getCore().getCoreDescriptor().getCoreContainer()
.getZkController().getZkClient();
}
@@ -54,9 +56,9 @@ public class ZKPropertiesWriter implemen
}
@Override
- public void persist(Properties props) {
- Properties existing = readIndexerProperties();
- existing.putAll(props);
+ public void persist(Map<String, Object> propObjs) {
+ Properties existing = mapToProperties(readIndexerProperties());
+ existing.putAll(mapToProperties(propObjs));
ByteArrayOutputStream output = new ByteArrayOutputStream();
try {
existing.store(output, "");
@@ -78,7 +80,7 @@ public class ZKPropertiesWriter implemen
}
@Override
- public Properties readIndexerProperties() {
+ public Map<String, Object> readIndexerProperties() {
Properties props = new Properties();
try {
byte[] data = zkClient.getData(path, null, null, false);
@@ -90,6 +92,6 @@ public class ZKPropertiesWriter implemen
log.warn(
"Could not read DIH properties from " + path + " :" + e.getClass(), e);
}
- return props;
+ return propertiesToMap(props);
}
}
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/ConfigNameConstants.java Mon Dec 10 16:36:47 2012
@@ -28,6 +28,8 @@ public class ConfigNameConstants {
public static final String NAME = "name";
public static final String PROCESSOR = "processor";
+
+ public static final String PROPERTY_WRITER = "propertyWriter";
/**
* @deprecated use IMPORTER_NS_SHORT instead
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java Mon Dec 10 16:36:47 2012
@@ -43,13 +43,18 @@ import org.w3c.dom.Element;
public class DIHConfiguration {
// TODO - remove from here and add it to entity
private final String deleteQuery;
+
private final List<Entity> entities;
private final String onImportStart;
private final String onImportEnd;
private final List<Map<String, String>> functions;
private final Script script;
private final Map<String, Map<String,String>> dataSources;
- public DIHConfiguration(Element element, DataImporter di, List<Map<String, String>> functions, Script script, Map<String, Map<String,String>> dataSources) {
+ private final PropertyWriter propertyWriter;
+
+ public DIHConfiguration(Element element, DataImporter di,
+ List<Map<String,String>> functions, Script script,
+ Map<String,Map<String,String>> dataSources, PropertyWriter pw) {
this.deleteQuery = ConfigParseUtil.getStringAttribute(element, "deleteQuery", null);
this.onImportStart = ConfigParseUtil.getStringAttribute(element, "onImportStart", null);
this.onImportEnd = ConfigParseUtil.getStringAttribute(element, "onImportEnd", null);
@@ -73,6 +78,7 @@ public class DIHConfiguration {
this.functions = Collections.unmodifiableList(modFunc);
this.script = script;
this.dataSources = Collections.unmodifiableMap(dataSources);
+ this.propertyWriter = pw;
}
public String getDeleteQuery() {
return deleteQuery;
@@ -95,4 +101,7 @@ public class DIHConfiguration {
public Script getScript() {
return script;
}
+ public PropertyWriter getPropertyWriter() {
+ return propertyWriter;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java (original)
+++ lucene/dev/branches/lucene4547/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java Mon Dec 10 16:36:47 2012
@@ -226,7 +226,7 @@ public class AbstractDIHCacheTestCase {
}
public static Context getContext(final Map<String, String> entityAttrs) {
- VariableResolverImpl resolver = new VariableResolverImpl();
+ VariableResolver resolver = new VariableResolver();
final Context delegate = new ContextImpl(null, resolver, null, null, new HashMap<String, Object>(), null, null);
return new TestContext(entityAttrs, delegate, null, true);
}