You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/03/11 20:17:51 UTC
svn commit: r752586 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
Author: shalin
Date: Wed Mar 11 19:17:50 2009
New Revision: 752586
URL: http://svn.apache.org/viewvc?rev=752586&view=rev
Log:
SOLR-1066 -- New methods in Context to expose Script details. ScriptTransformer changed to read scripts through the new API methods
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Wed Mar 11 19:17:50 2009
@@ -86,6 +86,10 @@
of TemplateTransformer in other transformers, among other things.
(Fergus McMenemie, Noble Paul via shalin)
+20.SOLR-1066: New methods in Context to expose Script details. ScriptTransformer changed to read scripts
+ through the new API methods.
+ (Noble Paul via shalin)
+
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java Wed Mar 11 19:17:50 2009
@@ -77,7 +77,7 @@
* Helper for creating a Context instance. Useful for testing Transformers
*/
@SuppressWarnings("unchecked")
- public static Context getContext(DataConfig.Entity parentEntity,
+ public static TestContext getContext(DataConfig.Entity parentEntity,
VariableResolverImpl resolver, DataSource parentDataSource,
int currProcess, final List<Map<String, String>> entityFields,
final Map<String, String> entityAttrs) {
@@ -85,65 +85,7 @@
final Context delegate = new ContextImpl(parentEntity, resolver,
parentDataSource, currProcess,
new HashMap<String, Object>(), null, null);
- return new Context() {
- public String getEntityAttribute(String name) {
- return entityAttrs == null ? delegate.getEntityAttribute(name)
- : entityAttrs.get(name);
- }
-
- public List<Map<String, String>> getAllEntityFields() {
- return entityFields == null ? delegate.getAllEntityFields()
- : entityFields;
- }
-
- public VariableResolver getVariableResolver() {
- return delegate.getVariableResolver();
- }
-
- public DataSource getDataSource() {
- return delegate.getDataSource();
- }
-
- public boolean isRootEntity() {
- return false;
- }
-
- public int currentProcess() {
- return delegate.currentProcess();
- }
-
- public Map<String, Object> getRequestParameters() {
- return delegate.getRequestParameters();
- }
-
- public EntityProcessor getEntityProcessor() {
- return null;
- }
-
- public void setSessionAttribute(String name, Object val, String scope) {
- delegate.setSessionAttribute(name, val, scope);
- }
-
- public Object getSessionAttribute(String name, String scope) {
- return delegate.getSessionAttribute(name, scope);
- }
-
- public Context getParentContext() {
- return delegate.getParentContext();
- }
-
- public DataSource getDataSource(String name) {
- return delegate.getDataSource(name);
- }
-
- public SolrCore getSolrCore() {
- return delegate.getSolrCore();
- }
-
- public Map<String, Object> getStats() {
- return delegate.getStats();
- }
- };
+ return new TestContext(entityAttrs, delegate, entityFields);
}
/**
@@ -162,4 +104,84 @@
return result;
}
+
+ static class TestContext extends Context {
+ private final Map<String, String> entityAttrs;
+ private final Context delegate;
+ private final List<Map<String, String>> entityFields;
+ String script,scriptlang;
+
+ public TestContext(Map<String, String> entityAttrs, Context delegate, List<Map<String, String>> entityFields) {
+ this.entityAttrs = entityAttrs;
+ this.delegate = delegate;
+ this.entityFields = entityFields;
+ }
+
+ public String getEntityAttribute(String name) {
+ return entityAttrs == null ? delegate.getEntityAttribute(name)
+ : entityAttrs.get(name);
+ }
+
+ public List<Map<String, String>> getAllEntityFields() {
+ return entityFields == null ? delegate.getAllEntityFields()
+ : entityFields;
+ }
+
+ public VariableResolver getVariableResolver() {
+ return delegate.getVariableResolver();
+ }
+
+ public DataSource getDataSource() {
+ return delegate.getDataSource();
+ }
+
+ public boolean isRootEntity() {
+ return false;
+ }
+
+ public int currentProcess() {
+ return delegate.currentProcess();
+ }
+
+ public Map<String, Object> getRequestParameters() {
+ return delegate.getRequestParameters();
+ }
+
+ public EntityProcessor getEntityProcessor() {
+ return null;
+ }
+
+ public void setSessionAttribute(String name, Object val, String scope) {
+ delegate.setSessionAttribute(name, val, scope);
+ }
+
+ public Object getSessionAttribute(String name, String scope) {
+ return delegate.getSessionAttribute(name, scope);
+ }
+
+ public Context getParentContext() {
+ return delegate.getParentContext();
+ }
+
+ public DataSource getDataSource(String name) {
+ return delegate.getDataSource(name);
+ }
+
+ public SolrCore getSolrCore() {
+ return delegate.getSolrCore();
+ }
+
+ public Map<String, Object> getStats() {
+ return delegate.getStats();
+ }
+
+
+ public String getScript() {
+ return script == null ? delegate.getScript() : script;
+ }
+
+ public String getScriptLanguage() {
+ return scriptlang == null ? delegate.getScriptLanguage() : scriptlang;
+ }
+ }
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java Wed Mar 11 19:17:50 2009
@@ -180,4 +180,14 @@
* @return a Map containing running statistics of the current import
*/
public abstract Map<String, Object> getStats();
+
+ /**
+ * Returns the text specified in the script tag in the data-config.xml
+ */
+ public abstract String getScript();
+
+ /**
+ * Returns the language of the script as specified in the script tag in data-config.xml
+ */
+ public abstract String getScriptLanguage();
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java Wed Mar 11 19:17:50 2009
@@ -179,4 +179,20 @@
public Map<String, Object> getStats() {
return docBuilder != null ? docBuilder.importStatistics.getStatsSnapshot() : Collections.<String, Object>emptyMap();
}
+
+ public String getScript() {
+ if(dataImporter != null) {
+ DataConfig.Script script = dataImporter.getConfig().script;
+ return script == null ? null : script.text;
+ }
+ return null;
+ }
+
+ public String getScriptLanguage() {
+ if (dataImporter != null) {
+ DataConfig.Script script = dataImporter.getConfig().script;
+ return script == null ? null : script.language;
+ }
+ return null;
+ }
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Wed Mar 11 19:17:50 2009
@@ -163,7 +163,7 @@
public static class Script {
public String language;
- public String script;
+ public String text;
public Script() {
}
@@ -173,7 +173,7 @@
StringBuilder buffer = new StringBuilder();
String script = getTxt(e, buffer);
if (script != null)
- this.script = script.trim();
+ this.text = script.trim();
}
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Wed Mar 11 19:17:50 2009
@@ -92,10 +92,6 @@
indexerNamespace.put(key, lastIndex);
}
}
- if (dataImporter.getConfig().script != null) {
- indexerNamespace.put(DataConfig.SCRIPT, dataImporter.getConfig().script.script);
- indexerNamespace.put(DataConfig.SCRIPT_LANG, dataImporter.getConfig().script.language);
- }
resolver.addNamespace(DataConfig.IMPORTER_NS, indexerNamespace);
return resolver;
}
@@ -290,7 +286,7 @@
Map<String, Object> pk, DataConfig.Entity entity, boolean isRoot,
ContextImpl parentCtx) {
- EntityProcessor entityProcessor = getEntityProcessor(entity, dataImporter.getCore());
+ EntityProcessor entityProcessor = getEntityProcessor(entity);
ContextImpl ctx = new ContextImpl(entity, vr, null,
pk == null ? Context.FULL_DUMP : Context.DELTA_DUMP,
@@ -482,7 +478,7 @@
}
}
- public static EntityProcessor getEntityProcessor(DataConfig.Entity entity, SolrCore core) {
+ private EntityProcessor getEntityProcessor(DataConfig.Entity entity) {
if (entity.processor != null)
return entity.processor;
EntityProcessor entityProcessor;
@@ -490,7 +486,7 @@
entityProcessor = new SqlEntityProcessor();
} else {
try {
- entityProcessor = (EntityProcessor) loadClass(entity.proc, core)
+ entityProcessor = (EntityProcessor) loadClass(entity.proc, dataImporter.getCore())
.newInstance();
} catch (Exception e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
@@ -511,7 +507,7 @@
@SuppressWarnings("unchecked")
public Set<Map<String, Object>> collectDelta(DataConfig.Entity entity,
DataConfig.Entity parentEntity, VariableResolverImpl resolver,
- DataImporter context, Set<Map<String, Object>> deletedRows) {
+ DataImporter dataImporter, Set<Map<String, Object>> deletedRows) {
//someone called abort
if (stop.get())
return new HashSet();
@@ -522,7 +518,7 @@
for (DataConfig.Entity entity1 : entity.entities) {
//this ensures that we start from the leaf nodes
- myModifiedPks.addAll(collectDelta(entity1, entity, resolver, context,
+ myModifiedPks.addAll(collectDelta(entity1, entity, resolver, dataImporter,
deletedRows));
//someone called abort
if (stop.get())
@@ -534,7 +530,7 @@
Set<Map<String, Object>> deltaSet = new HashSet<Map<String, Object>>();
resolver.addNamespace(null, (Map) entity.allAttributes);
- EntityProcessor entityProcessor = getEntityProcessor(entity, context.getCore());
+ EntityProcessor entityProcessor = getEntityProcessor(entity);
ContextImpl context1 = new ContextImpl(entity, resolver, null, Context.FIND_DELTA, session, null, this);
resolver.context = context1;
entityProcessor.init(context1);
@@ -585,7 +581,7 @@
//all that we have captured is useless (in a sub-entity) if no rows in the parent is modified because of these
//so propogate up the changes in the chain
if (parentEntity != null && parentEntity.isDocRoot) {
- EntityProcessor parentEntityProcessor = getEntityProcessor(parentEntity, context.getCore());
+ EntityProcessor parentEntityProcessor = getEntityProcessor(parentEntity);
ContextImpl context2 = new ContextImpl(parentEntity, resolver, null, Context.FIND_DELTA, session, null, this);
resolver.context = context2;
parentEntityProcessor.init(context2);
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java Wed Mar 11 19:17:50 2009
@@ -71,10 +71,8 @@
private void initEngine(Context context) {
try {
- String scriptText = (String) context.getVariableResolver().resolve(
- DataConfig.IMPORTER_NS + "." + DataConfig.SCRIPT);
- String scriptLang = (String) context.getVariableResolver().resolve(
- DataConfig.IMPORTER_NS + "." + DataConfig.SCRIPT_LANG);
+ String scriptText = context.getScript();
+ String scriptLang = context.getScriptLanguage();
Object scriptEngineMgr = Class
.forName("javax.script.ScriptEngineManager").newInstance();
// create a Script engine
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java?rev=752586&r1=752585&r2=752586&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java Wed Mar 11 19:17:50 2009
@@ -64,14 +64,11 @@
Map<String, String> entity = new HashMap<String, String>();
entity.put("name", "hello");
entity.put("transformer", "script:" + funcName);
- Map<String, Object> dataImporterNs = new HashMap<String, Object>();
- dataImporterNs.put(DataConfig.SCRIPT_LANG, "JavaScript");
- dataImporterNs.put(DataConfig.SCRIPT, script);
- VariableResolverImpl vr = new VariableResolverImpl();
- vr.addNamespace(DataConfig.IMPORTER_NS, dataImporterNs);
- Context context = AbstractDataImportHandlerTest.getContext(null, vr, null,
+ AbstractDataImportHandlerTest.TestContext context = AbstractDataImportHandlerTest.getContext(null, null, null,
0, fields, entity);
+ context.script = script;
+ context.scriptlang = "JavaScript";
return context;
}
@@ -101,7 +98,7 @@
DataConfig config = new DataConfig();
config.readFromXml((Element) document.getElementsByTagName("dataConfig")
.item(0));
- Assert.assertTrue(config.script.script.indexOf("checkNextToken") > -1);
+ Assert.assertTrue(config.script.text.indexOf("checkNextToken") > -1);
}
@Test
@@ -114,7 +111,7 @@
config.readFromXml((Element) document.getElementsByTagName("dataConfig")
.item(0));
- Context c = getContext("checkNextToken", config.script.script);
+ Context c = getContext("checkNextToken", config.script.text);
Map map = new HashMap();
map.put("nextToken", "hello");