You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2011/07/21 12:56:55 UTC
svn commit: r1149108 - in /lucene/dev/trunk/solr/contrib/dataimporthandler:
./ src/java/org/apache/solr/handler/dataimport/
src/test/org/apache/solr/handler/dataimport/
Author: shalin
Date: Thu Jul 21 10:56:54 2011
New Revision: 1149108
URL: http://svn.apache.org/viewvc?rev=1149108&view=rev
Log:
SOLR-2655 -- DIH multi threaded mode does not resolve attributes correctly
Added:
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java
Modified:
lucene/dev/trunk/solr/contrib/dataimporthandler/CHANGES.txt
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/CHANGES.txt?rev=1149108&r1=1149107&r2=1149108&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/CHANGES.txt Thu Jul 21 10:56:54 2011
@@ -19,6 +19,7 @@ Bug Fixes
* SOLR-2644: When using threads=2 the default logging is set too high (Bill Bell via shalin)
* SOLR-2492: DIH does not commit if only deletes are processed (James Dyer via shalin)
* SOLR-2186: DataImportHandler's multi-threaded option throws NPE (Lance Norskog, Frank Wesemann, shalin)
+* SOLR-2655: DIH multi threaded mode does not resolve attributes correctly (Frank Wesemann, shalin)
================== 3.3.0 ==================
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1149108&r1=1149107&r2=1149108&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Thu Jul 21 10:56:54 2011
@@ -85,7 +85,8 @@ public class DocBuilder {
public VariableResolverImpl getVariableResolver() {
try {
VariableResolverImpl resolver = null;
- if(dataImporter != null && dataImporter.getCore() != null){
+ if(dataImporter != null && dataImporter.getCore() != null
+ && dataImporter.getCore().getResourceLoader().getCoreProperties() != null){
resolver = new VariableResolverImpl(dataImporter.getCore().getResourceLoader().getCoreProperties());
} else resolver = new VariableResolverImpl();
Map<String, Object> indexerNamespace = new HashMap<String, Object>();
@@ -436,11 +437,11 @@ public class DocBuilder {
private void runAThread(ThreadedEntityProcessorWrapper epw, EntityRow rows, String currProcess) throws Exception {
currentEntityProcWrapper.set(epw);
epw.threadedInit(context);
- initEntity();
try {
+ Context.CURRENT_CONTEXT.set(context);
epw.init(rows);
+ initEntity();
DocWrapper docWrapper = this.docWrapper;
- Context.CURRENT_CONTEXT.set(context);
for (; ;) {
if(DocBuilder.this.stop.get()) break;
try {
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java?rev=1149108&r1=1149107&r2=1149108&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EvaluatorBag.java Thu Jul 21 10:56:54 2011
@@ -217,7 +217,6 @@ public class EvaluatorBag {
Evaluator evaluator = evaluators.get(fname);
if (evaluator == null)
return null;
- VariableResolverImpl vri = VariableResolverImpl.CURRENT_VARIABLE_RESOLVER.get();
return evaluator.evaluate(m.group(2), Context.CURRENT_CONTEXT.get());
}
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java?rev=1149108&r1=1149107&r2=1149108&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java Thu Jul 21 10:56:54 2011
@@ -35,11 +35,6 @@ import java.util.regex.Pattern;
public class VariableResolverImpl extends VariableResolver {
private Map<String, Object> container = new HashMap<String, Object>();
- /**
- * Used for creating Evaluators
- */
- Context context;
-
private final TemplateString templateString = new TemplateString();
private final Map defaults ;
Added: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java?rev=1149108&view=auto
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java (added)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java Thu Jul 21 10:56:54 2011
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Test DocBuilder with "threads"
+ */
+public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ List<Map<String, Object>> docs = new ArrayList<Map<String, Object>>();
+ docs.add(createMap("id", "1", "worker", "one"));
+ docs.add(createMap("id", "2", "worker", "two"));
+ docs.add(createMap("id", "3", "worker", "three"));
+ docs.add(createMap("id", "4", "worker", "four"));
+ MockDataSource.setIterator("select * from y", docs.iterator());
+ for (Map<String, Object> aDoc : docs) {
+ String theWorker = (String) aDoc.get("worker");
+ final List<Map<String, Object>> details = getDetails4Worker(theWorker);
+ log.info("details: " + details);
+ MockDataSource.setIterator(theWorker, details.iterator());
+ }
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ DemoProcessor.entitiesInitied = 0;
+ DemoEvaluator.evaluated = 0;
+ MockDataSource.clearCache();
+ super.tearDown();
+ }
+
+ @Test
+ public void testProcessorThreaded2Entities() throws Exception {
+ runFullImport(threaded2EntitiesWithProcessor);
+ assertEquals("EntityProcessor.init() for child entity was called less times than the number of rows",
+ 4, DemoProcessor.entitiesInitied);
+ }
+
+ @Test
+ public void testProcessor2EntitiesNoThreads() throws Exception {
+ runFullImport(twoEntitiesWithProcessor);
+ assertEquals("EntityProcessor.init() for child entity was called less times than the number of rows",
+ 4, DemoProcessor.entitiesInitied);
+ }
+
+ /*
+ * This test fails in TestEnviroment, but works in real Live
+ */
+ @Test
+ public void testEvaluator() throws Exception {
+ runFullImport(twoEntitiesWithEvaluatorProcessor);
+ assertEquals("Evaluator was invoked less times than the number of rows",
+ 4, DemoEvaluator.evaluated);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<Map<String, Object>> getDetails4Worker(String aWorker) {
+ List<Map<String, Object>> details4Worker = new ArrayList<Map<String, Object>>();
+ details4Worker.add(createMap("author_s", "Author_" + aWorker, "title_s", "Title for " + aWorker, "text_s", " Text for " + aWorker));
+ return details4Worker;
+ }
+
+ private final String threaded2EntitiesWithProcessor =
+
+ "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ "<document>" +
+ "<entity name=\"job\" query=\"select * from y\"" +
+ " pk=\"id\" \n" +
+ " threads='1'\n" +
+ ">" +
+ "<field column=\"id\" />\n" +
+ "<entity name=\"details\" processor=\"TestDocBuilderThreaded$DemoProcessor\" \n" +
+ "worker=\"${job.worker}\" \n" +
+ "query=\"${job.worker}\" \n" +
+ "transformer=\"TemplateTransformer\" " +
+ " >" +
+ "<field column=\"author_s\" />" +
+ "<field column=\"title_s\" />" +
+ " <field column=\"text_s\" />" +
+ " <field column=\"generated_id_s\" template=\"generated_${job.id}\" />" +
+ "</entity>" +
+ "</entity>" +
+ "</document>" +
+ "</dataConfig>";
+
+ private final String twoEntitiesWithProcessor =
+
+ "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ "<document>" +
+ "<entity name=\"job\" query=\"select * from y\"" +
+ " pk=\"id\" \n" +
+ ">" +
+ "<field column=\"id\" />\n" +
+ "<entity name=\"details\" processor=\"TestDocBuilderThreaded$DemoProcessor\" \n" +
+ "worker=\"${job.worker}\" \n" +
+ "query=\"${job.worker}\" \n" +
+ "transformer=\"TemplateTransformer\" " +
+ " >" +
+ "<field column=\"author_s\" />" +
+ "<field column=\"title_s\" />" +
+ " <field column=\"text_s\" />" +
+ " <field column=\"generated_id_s\" template=\"generated_${job.id}\" />" +
+ "</entity>" +
+ "</entity>" +
+ "</document>" +
+ "</dataConfig>";
+
+ private final String twoEntitiesWithEvaluatorProcessor =
+
+ "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ "<function name=\"concat\" class=\"TestDocBuilderThreaded$DemoEvaluator\" />" +
+ "<document>" +
+ "<entity name=\"job\" query=\"select * from y\"" +
+ " pk=\"id\" \n" +
+ " threads=\"1\" " +
+ ">" +
+ "<field column=\"id\" />\n" +
+ "<entity name=\"details\" processor=\"TestDocBuilderThreaded$DemoProcessor\" \n" +
+ "worker=\"${dataimporter.functions.concat(details.author_s, ':_:' , details.title_s, 9 )}\" \n" +
+ "query=\"${job.worker}\" \n" +
+ "transformer=\"TemplateTransformer\" " +
+ " >" +
+ "<field column=\"author_s\" />" +
+ "<field column=\"title_s\" />" +
+ " <field column=\"text_s\" />" +
+ " <field column=\"generated_id_s\" template=\"generated_${job.id}\" />" +
+ "</entity>" +
+ "</entity>" +
+ "</document>" +
+ "</dataConfig>";
+
+
+ public static class DemoProcessor extends SqlEntityProcessor {
+
+ public static int entitiesInitied = 0;
+
+ @Override
+ public void init(Context context) {
+ super.init(context);
+ String result = context.getResolvedEntityAttribute("worker");
+ if (result == null || result.trim().length() == 0) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Could not resolve entity attribute");
+ } else entitiesInitied++;
+ }
+ }
+
+ public static class DemoEvaluator extends Evaluator {
+ public static int evaluated = 0;
+
+ /* (non-Javadoc)
+ * @see org.apache.solr.handler.dataimport.Evaluator#evaluate(java.lang.String, org.apache.solr.handler.dataimport.Context)
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public String evaluate(String expression, Context context) {
+ List allParams = EvaluatorBag.parseParams(expression, context.getVariableResolver());
+ StringBuilder result = new StringBuilder();
+ for (Object aVar : allParams) {
+ result.append(aVar.toString());
+ }
+ evaluated++;
+ return result.toString();
+ }
+ }
+}