You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/12/30 07:42:48 UTC
svn commit: r730055 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Mon Dec 29 22:42:48 2008
New Revision: 730055
URL: http://svn.apache.org/viewvc?rev=730055&view=rev
Log:
SOLR-782 followup -- Removed multiple document nodes in the configuration xml. Removed support for 'default' variables, they are automatically available as request parameters.
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Mon Dec 29 22:42:48 2008
@@ -93,7 +93,9 @@
Other
----------------------
-1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument
+1. SOLR-782: Refactored SolrWriter to make it a concrete class and removed wrappers over SolrInputDocument.
+ Refactored to load Evaluators lazily. Removed multiple document nodes in the configuration xml.
+ Removed support for 'default' variables, they are automatically available as request parameters.
(Noble Paul via shalin)
================== Release 1.3.0 20080915 ==================
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Mon Dec 29 22:42:48 2008
@@ -41,9 +41,7 @@
* @since solr 1.3
*/
public class DataConfig {
- public List<Document> documents;
-
- private Map<String, Document> documentCache;
+ public Document document;
public List<Map<String, String >> functions = new ArrayList<Map<String ,String>>();
@@ -53,20 +51,7 @@
public Map<String, SchemaField> lowerNameVsSchemaField = new HashMap<String, SchemaField>();
- public Document getDocumentByName(String name) {
- if (documentCache == null) {
- documentCache = new HashMap<String, Document>();
- for (Document document : documents)
- documentCache.put(document.name, document);
- }
-
- return documentCache.get(name);
- }
-
public static class Document {
- // TODO - remove this
- public String name;
-
// TODO - remove from here and add it to entity
public String deleteQuery;
@@ -76,7 +61,6 @@
}
public Document(Element element) {
- this.name = getStringAttribute(element, NAME, null);
this.deleteQuery = getStringAttribute(element, "deleteQuery", null);
List<Element> l = getChildNodes(element, "entity");
for (Element e : l)
@@ -222,10 +206,11 @@
public void readFromXml(Element e) {
List<Element> n = getChildNodes(e, "document");
- if (!n.isEmpty())
- documents = new ArrayList<Document>();
- for (Element element : n)
- documents.add(new Document(element));
+ if (n.isEmpty()) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "DataImportHandler " +
+ "configuration file must have one <document> node.");
+ }
+ document = new Document(n.get(0));
n = getChildNodes(e, SCRIPT);
if (!n.isEmpty()) {
@@ -307,10 +292,8 @@
}
public void clearCaches() {
- for (Document document : documents)
- for (Entity entity : document.entities)
- entity.clearCache();
-
+ for (Entity entity : document.entities)
+ entity.clearCache();
}
public static final String SCRIPT = "script";
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java Mon Dec 29 22:42:48 2008
@@ -161,7 +161,7 @@
}
if (command != null && DataImporter.ABORT_CMD.equals(command)) {
- importer.runCmd(requestParams, null, null);
+ importer.runCmd(requestParams, null);
} else if (importer.isBusy()) {
message = DataImporter.MSG.CMD_RUNNING;
} else if (command != null) {
@@ -177,7 +177,7 @@
if (requestParams.debug) {
if (debugEnabled) {
// Synchronous request for the debug mode
- importer.runCmd(requestParams, sw, variables);
+ importer.runCmd(requestParams, sw);
rsp.add("mode", "debug");
rsp.add("documents", debugDocuments);
if (sw.debugLogger != null)
@@ -188,7 +188,7 @@
}
} else {
// Asynchronous request for normal mode
- importer.runAsync(requestParams, sw, variables);
+ importer.runAsync(requestParams, sw);
}
} else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
importer = null;
@@ -222,8 +222,7 @@
@SuppressWarnings("unchecked")
private void processConfiguration(NamedList defaults) {
if (defaults == null) {
- LOG
- .info("No configuration specified in solrconfig.xml for DataImportHandler");
+ LOG.info("No configuration specified in solrconfig.xml for DataImportHandler");
return;
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Mon Dec 29 22:42:48 2008
@@ -95,15 +95,13 @@
config.lowerNameVsSchemaField.put(entry.getKey().toLowerCase(), entry.getValue());
}
- for (DataConfig.Document document : config.documents) {
- for (DataConfig.Entity e : document.entities) {
- Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
- initEntity(e, fields, false);
- String errs = verifyWithSchema(fields);
- if (errs != null) {
- throw new DataImportHandlerException(
- DataImportHandlerException.SEVERE, errs);
- }
+ for (DataConfig.Entity e : config.document.entities) {
+ Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
+ initEntity(e, fields, false);
+ String errs = verifyWithSchema(fields);
+ if (errs != null) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, errs);
}
}
}
@@ -150,8 +148,9 @@
void loadAndInit(String configStr) {
loadDataConfig(configStr);
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
- DataConfig.Entity e = getConfig().documents.get(0).entities.get(0);
- initEntity(e, fields, false);
+ for (DataConfig.Entity entity : config.document.entities) {
+ initEntity(entity, fields, false);
+ }
}
void loadDataConfig(String configFile) {
@@ -308,8 +307,7 @@
return importLock.isLocked();
}
- public void doFullImport(SolrWriter writer, RequestParams requestParams,
- Map<String, String> variables) {
+ public void doFullImport(SolrWriter writer, RequestParams requestParams) {
LOG.info("Starting Full Import");
setStatus(Status.RUNNING_FULL_DUMP);
@@ -319,8 +317,8 @@
try {
if (requestParams.clean)
writer.doDeleteAll();
- docBuilder = new DocBuilder(this, writer, requestParams, variables);
- docBuilder.execute(getConfig().documents.get(0).name);
+ docBuilder = new DocBuilder(this, writer, requestParams);
+ docBuilder.execute();
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (Throwable t) {
@@ -334,8 +332,7 @@
}
- public void doDeltaImport(SolrWriter writer, RequestParams requestParams,
- Map<String, String> variables) {
+ public void doDeltaImport(SolrWriter writer, RequestParams requestParams) {
LOG.info("Starting Delta Import");
setStatus(Status.RUNNING_DELTA_DUMP);
@@ -343,8 +340,8 @@
if (requestParams.commit) {
setIndexStartTime(new Date());
}
- docBuilder = new DocBuilder(this, writer, requestParams, variables);
- docBuilder.execute(config.documents.get(0).name);
+ docBuilder = new DocBuilder(this, writer, requestParams);
+ docBuilder.execute();
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (Throwable t) {
@@ -358,17 +355,16 @@
}
- public void runAsync(final RequestParams reqParams, final SolrWriter sw,
- final Map<String, String> variables) {
+ public void runAsync(final RequestParams reqParams, final SolrWriter sw) {
new Thread() {
@Override
public void run() {
- runCmd(reqParams, sw, variables);
+ runCmd(reqParams, sw);
}
}.start();
}
- void runCmd(RequestParams reqParams, SolrWriter sw, Map<String, String> variables) {
+ void runCmd(RequestParams reqParams, SolrWriter sw) {
String command = reqParams.command;
if (command.equals(ABORT_CMD)) {
if (docBuilder != null) {
@@ -382,9 +378,9 @@
Date lastModified = sw.loadIndexStartTime();
setLastIndexTime(lastModified);
if (command.equals("full-import")) {
- doFullImport(sw, reqParams, variables);
+ doFullImport(sw, reqParams);
} else if (command.equals(DELTA_IMPORT_CMD)) {
- doDeltaImport(sw, reqParams, variables);
+ doDeltaImport(sw, reqParams);
}
} finally {
importLock.unlock();
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Mon Dec 29 22:42:48 2008
@@ -59,50 +59,43 @@
boolean verboseDebug = false;
- private Map<String, String> defaultVariables;
-
private Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
- public DocBuilder(DataImporter context, SolrWriter writer,
- DataImporter.RequestParams reqParams, Map<String, String> variables) {
+ public DocBuilder(DataImporter context, SolrWriter writer, DataImporter.RequestParams reqParams) {
INSTANCE.set(this);
this.dataImporter = context;
this.writer = writer;
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
requestParameters = reqParams;
verboseDebug = requestParameters.debug && requestParameters.verbose;
- defaultVariables = Collections.unmodifiableMap(variables);
}
- public VariableResolverImpl getVariableResolver(DataImporter context) {
+ public VariableResolverImpl getVariableResolver() {
VariableResolverImpl resolver = new VariableResolverImpl();
Map<String, Object> indexerNamespace = new HashMap<String, Object>();
- if (context.getLastIndexTime() != null)
+ if (dataImporter.getLastIndexTime() != null)
indexerNamespace.put(LAST_INDEX_TIME, DataImporter.DATE_TIME_FORMAT
- .format(context.getLastIndexTime()));
- indexerNamespace.put(INDEX_START_TIME, context.getIndexStartTime());
+ .format(dataImporter.getLastIndexTime()));
+ indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
indexerNamespace.put("request", requestParameters.requestParams);
- indexerNamespace.put("defaults", defaultVariables);
indexerNamespace.put("functions", EvaluatorBag.getFunctionsNamespace(resolver,
dataImporter.getConfig().functions, this));
- if (context.getConfig().script != null) {
+ if (dataImporter.getConfig().script != null) {
indexerNamespace
- .put(DataConfig.SCRIPT, context.getConfig().script.script);
+ .put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
indexerNamespace.put(DataConfig.SCRIPT_LANG,
- context.getConfig().script.language);
+ dataImporter.getConfig().script.language);
}
resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
return resolver;
}
@SuppressWarnings("unchecked")
- public void execute(String docName) {
+ public void execute() {
dataImporter.store(DataImporter.STATUS_MSGS, statusMessages);
- document = dataImporter.getConfig().getDocumentByName(docName);
- if (document == null)
- return;
+ document = dataImporter.getConfig().document;
final AtomicLong startTime = new AtomicLong(System.currentTimeMillis());
statusMessages.put(TIME_ELAPSED, new Object() {
public String toString() {
@@ -180,14 +173,14 @@
@SuppressWarnings("unchecked")
private void doFullDump() {
addStatusMessage("Full Dump Started");
- buildDocument(getVariableResolver(dataImporter), null, null, root, true,
+ buildDocument(getVariableResolver(), null, null, root, true,
null);
}
@SuppressWarnings("unchecked")
private void doDelta() {
addStatusMessage("Delta Dump started");
- VariableResolverImpl resolver = getVariableResolver(dataImporter);
+ VariableResolverImpl resolver = getVariableResolver();
if (document.deleteQuery != null) {
writer.deleteByQuery(document.deleteQuery);
@@ -211,7 +204,7 @@
deletedKeys = null;
statusMessages.put("Total Changed Documents", allPks.size());
- VariableResolverImpl vri = getVariableResolver(dataImporter);
+ VariableResolverImpl vri = getVariableResolver();
Iterator<Map<String, Object>> pkIter = allPks.iterator();
while (pkIter.hasNext()) {
Map<String, Object> map = pkIter.next();
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java Mon Dec 29 22:42:48 2008
@@ -75,8 +75,7 @@
DataConfig dc = new DataConfig();
dc.readFromXml(doc.getDocumentElement());
- Assert.assertEquals("atrimlisting",
- dc.documents.get(0).entities.get(0).name);
+ Assert.assertEquals("atrimlisting", dc.document.entities.get(0).name);
}
private static final String xml = "<dataConfig>\n"
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java Mon Dec 29 22:42:48 2008
@@ -48,14 +48,14 @@
DataImporter di = new DataImporter();
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
- DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
+ DataConfig.Entity ent = cfg.document.entities.get(0);
MockDataSource.setIterator("select * from x", new ArrayList().iterator());
ent.dataSrc = new MockDataSource();
ent.isDocRoot = true;
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi, Collections.EMPTY_MAP);
+ di.runCmd(rp, swi);
Assert.assertEquals(Boolean.TRUE, swi.deleteAllCalled);
Assert.assertEquals(Boolean.TRUE, swi.commitCalled);
Assert.assertEquals(0, swi.docs.size());
@@ -76,7 +76,7 @@
DataImporter di = new DataImporter();
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
- DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
+ DataConfig.Entity ent = cfg.document.entities.get(0);
List l = new ArrayList();
l.add(createMap("id", 1, "desc", "one"));
MockDataSource.setIterator("select * from x", l.iterator());
@@ -85,7 +85,7 @@
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi, Collections.EMPTY_MAP);
+ di.runCmd(rp, swi);
Assert.assertEquals(Boolean.TRUE, swi.deleteAllCalled);
Assert.assertEquals(Boolean.TRUE, swi.commitCalled);
Assert.assertEquals(1, swi.docs.size());
@@ -115,7 +115,7 @@
DataImporter di = new DataImporter();
di.loadDataConfig(dc_singleEntity);
DataConfig cfg = di.getConfig();
- DataConfig.Entity ent = cfg.documents.get(0).entities.get(0);
+ DataConfig.Entity ent = cfg.document.entities.get(0);
ent.isDocRoot = true;
DataImporter.RequestParams rp = new DataImporter.RequestParams();
rp.command = "full-import";
@@ -127,7 +127,7 @@
MockDataSource.setIterator("select * from x", l.iterator());
ent.dataSrc = new MockDataSource();
SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi, Collections.EMPTY_MAP);
+ di.runCmd(rp, swi);
Assert.assertEquals(Boolean.TRUE, swi.deleteAllCalled);
Assert.assertEquals(Boolean.TRUE, swi.commitCalled);
Assert.assertEquals(3, swi.docs.size());
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java?rev=730055&r1=730054&r2=730055&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFieldReader.java Mon Dec 29 22:42:48 2008
@@ -43,7 +43,7 @@
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(createMap("xml", xml));
MockDataSource.setIterator("select * from a", l.iterator());
- di.runCmd(rp, sw, new HashMap<String, String>());
+ di.runCmd(rp, sw);
Assert.assertEquals(sw.docs.get(0).getFieldValue("y"), "Hello");
MockDataSource.clearCache();
}