You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jd...@apache.org on 2012/04/03 18:01:15 UTC
svn commit: r1309004 - in
/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler: ./
src/java/org/apache/solr/handler/dataimport/
src/test-files/dih/TestMultiThreadedFileReader/
src/test-files/dih/TestMultiThreadedFileReader/TestDir1/ src/test-f...
Author: jdyer
Date: Tue Apr 3 16:01:15 2012
New Revision: 1309004
URL: http://svn.apache.org/viewvc?rev=1309004&view=rev
Log:
SOLR-3307: fix regression with multi-threaded FileListEntityProcessor (introduced w/SOLR-3011)
Added:
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir1/
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir1/testfile2.xml (with props)
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir2/
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir2/testfile1.xml (with props)
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestMultiThreadedFileReader.java (with props)
Modified:
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestThreaded.java
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt?rev=1309004&r1=1309003&r2=1309004&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt Tue Apr 3 16:01:15 2012
@@ -38,8 +38,8 @@ Bug Fixes
----------------------
* SOLR-2875: Fix the incorrect url in tika-data-config.xml (Shinichiro Abe via koji)
-* SOLR-3011: Extensive bug fixes for "threads" parameter. Also added deprecation a warning when using "threads".
- (Mikhail Khludnev via James Dyer)
+* SOLR-3011 / SOLR-3307: Extensive bug fixes for "threads" parameter. Also added deprecation a warning when using
+ "threads". (Mikhail Khludnev, Bernd Fehling, James Dyer)
================== 3.4.0 ==================
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java?rev=1309004&r1=1309003&r2=1309004&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java Tue Apr 3 16:01:15 2012
@@ -50,15 +50,16 @@ public class EntityProcessorBase extends
protected DIHCacheSupport cacheSupport = null;
+
@Override
public void init(Context context) {
this.context = context;
if (isFirstInit) {
firstInit(context);
}
- rowIterator = null;
- query = null;
if(cacheSupport!=null) {
+ rowIterator = null;
+ query = null;
cacheSupport.initNewParent(context);
}
@@ -118,10 +119,12 @@ public class EntityProcessorBase extends
if (rowIterator.hasNext())
return rowIterator.next();
query = null;
+ rowIterator = null;
return null;
} catch (Exception e) {
SolrException.log(log, "getNext() failed for query '" + query + "'", e);
query = null;
+ rowIterator = null;
wrapAndThrow(DataImportHandlerException.WARN, e);
return null;
}
Added: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir1/testfile2.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir1/testfile2.xml?rev=1309004&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir1/testfile2.xml (added)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir1/testfile2.xml Tue Apr 3 16:01:15 2012
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<documents>
+ <document id="testdir2_testfile2_record1">
+ <element name="desc"><value>test desc 2_2_1</value></element>
+ </document>
+ <document id="testdir2_testfile2_record2">
+ <element name="desc"><value>test desc 2_2_2</value></element>
+ </document>
+ <document id="testdir2_testfile2_record3">
+ <element name="desc"><value>test desc 2_2_3</value></element>
+ </document>
+</documents>
Added: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir2/testfile1.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir2/testfile1.xml?rev=1309004&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir2/testfile1.xml (added)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test-files/dih/TestMultiThreadedFileReader/TestDir2/testfile1.xml Tue Apr 3 16:01:15 2012
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<documents>
+ <document id="testdir1_testfile1_record1">
+ <element name="desc"><value>test desc 1_1_1</value></element>
+ </document>
+ <document id="testdir1_testfile1_record2">
+ <element name="desc"><value>test desc 1_1_2</value></element>
+ </document>
+ <document id="testdir1_testfile1_record3">
+ <element name="desc"><value>test desc 1_1_3</value></element>
+ </document>
+</documents>
Added: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestMultiThreadedFileReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestMultiThreadedFileReader.java?rev=1309004&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestMultiThreadedFileReader.java (added)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestMultiThreadedFileReader.java Tue Apr 3 16:01:15 2012
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMultiThreadedFileReader extends AbstractDataImportHandlerTestCase {
+ private String getConf() {
+ int numThreads = random.nextInt(8) + 2; // between 2 and 10
+ System.out.println(getFile("dih/TestMultiThreadedFileReader").getAbsolutePath());
+ return
+ "<dataConfig>" +
+ " <dataSource name=\"files\" type=\"FileDataSource\" encoding=\"UTF-8\"/>" +
+ " <document>" +
+ " <entity name=\"filesdata\" processor=\"FileListEntityProcessor\" threads=\"" + numThreads + "\" rootEntity=\"false\" fileName=\"\\.xml$\" recursive=\"true\" dataSource=\"null\" baseDir=\"" + getFile("dih/TestMultiThreadedFileReader").getAbsolutePath() + "\" >" +
+ " <entity name=\"records\" processor=\"XPathEntityProcessor\" rootEntity=\"true\" dataSource=\"files\" stream=\"true\" forEach=\"/documents/document\" url=\"${filesdata.fileAbsolutePath}\">" +
+ " <field column=\"id\" xpath=\"/documents/document/@id\"/>" +
+ " <field column=\"desc\" xpath=\"/documents/document/element[@name='desc']/value\"/>" +
+ " </entity>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>"
+ ;
+ }
+ private String[] tests = {
+ "//*[@numFound='6']"
+ };
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ public void testMultiThreadedFileReader() throws Exception {
+ runFullImport(getConf());
+ assertQ(req("*:*"), tests );
+ }
+
+}
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestThreaded.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestThreaded.java?rev=1309004&r1=1309003&r2=1309004&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestThreaded.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestThreaded.java Tue Apr 3 16:01:15 2012
@@ -97,7 +97,7 @@ public class TestThreaded extends Abstra
@Test
public void testCachedThread_FullImport() throws Exception {
- int numThreads = random.nextInt(9) + 1; // between one and 10
+ int numThreads = random.nextInt(8) + 2; // between 2 and 10
String config = getCachedConfig(random.nextBoolean(), random.nextBoolean(), numThreads);
runFullImport(config);
}