You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Pankaj Sonawane <pa...@gmail.com> on 2015/01/13 08:31:24 UTC
Solr limiting number of rows to indexed to 21500 every time.
Hi,
I am using Solr DataImportHandler to index data from database
table(Oracle). One of the column contains String representation of XML
(Sample below).
*<options>*
*<option name="A">1</option>*
*<option name="B">2</option>*
*<option name="C">3</option>*
*.*
*.*
*.*
*</options> //<option> can be 100-200*
I want solr to index each 'name' in 'option' tag against its value
ex. JSON for 1 row
"docs": [ {
"COL1": "F",
"COL2": "ASDF", "COL3": "ATCC", "COL4": 29039757, "A_s": "1", "B_s": "2", "
C_s": "3",
.
.
.
* }*
// appending '_s' to 'name' attribute for making dynamic fields.
But while indexing data, *every time only 21500 rows get indexed*. After
these much records get indexed I got following exception:
*1320927 [Thread-15] ERROR
org.apache.solr.handler.dataimport.EntityProcessorBase û getNext() failed
for query 'SELECT col1,col2,col3,col4,XMLSERIALIZE(col5 AS CLOB) AS col5
FROM
tableName':org.apache.solr.handler.dataimport.DataImportHandlerException:
java.sql.SQLRecoverableException: No more data to read from socket*
* at
org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow(DataImportHandlerException.java:63)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:378)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.access$600(JdbcDataSource.java:258)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator$1.hasNext(JdbcDataSource.java:293)*
* at
org.apache.solr.handler.dataimport.EntityProcessorBase.getNext(EntityProcessorBase.java:116)*
* at
org.apache.solr.handler.dataimport.SqlEntityProcessor.nextRow(SqlEntityProcessor.java:75)*
* at
org.apache.solr.handler.dataimport.EntityProcessorWrapper.nextRow(EntityProcessorWrapper.java:243)*
* at
org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:476)*
* at
org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:415)*
* at
org.apache.solr.handler.dataimport.DocBuilder.doFullDump(DocBuilder.java:330)*
* at
org.apache.solr.handler.dataimport.DocBuilder.execute(DocBuilder.java:232)*
* at
org.apache.solr.handler.dataimport.DataImporter.doFullImport(DataImporter.java:416)*
* at
org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:480)*
* at
org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:461)*
*Caused by: java.sql.SQLRecoverableException: No more data to read from
socket*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1200)*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1865)*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1757)*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1750)*
* at
oracle.jdbc.driver.T4CClobAccessor.handlePrefetch(T4CClobAccessor.java:543)*
* at
oracle.jdbc.driver.T4CClobAccessor.unmarshalOneRow(T4CClobAccessor.java:197)*
* at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:916)*
* at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:835)*
* at oracle.jdbc.driver.T4C8Oall.readRXD(T4C8Oall.java:664)*
* at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:328)*
* at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:186)*
* at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:521)*
* at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:194)*
* at oracle.jdbc.driver.T4CStatement.fetch(T4CStatement.java:1074)*
* at
oracle.jdbc.driver.OracleResultSetImpl.close_or_fetch_from_next(OracleResultSetImpl.java:369)*
* at
oracle.jdbc.driver.OracleResultSetImpl.next(OracleResultSetImpl.java:273)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:370)*
* ... 12 more*
*1320928 [Thread-15] ERROR org.apache.solr.handler.dataimport.DocBuilder û
Exception while processing: e1 document : SolrInputDocument(fields:
[]):org.apache.solr.handler.dataimport.DataImportHandlerException:
java.sql.SQLRecoverableException: No more data to read from socket*
* at
org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow(DataImportHandlerException.java:63)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:378)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.access$600(JdbcDataSource.java:258)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator$1.hasNext(JdbcDataSource.java:293)*
* at
org.apache.solr.handler.dataimport.EntityProcessorBase.getNext(EntityProcessorBase.java:116)*
* at
org.apache.solr.handler.dataimport.SqlEntityProcessor.nextRow(SqlEntityProcessor.java:75)*
* at
org.apache.solr.handler.dataimport.EntityProcessorWrapper.nextRow(EntityProcessorWrapper.java:243)*
* at
org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:476)*
* at
org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:415)*
* at
org.apache.solr.handler.dataimport.DocBuilder.doFullDump(DocBuilder.java:330)*
* at
org.apache.solr.handler.dataimport.DocBuilder.execute(DocBuilder.java:232)*
* at
org.apache.solr.handler.dataimport.DataImporter.doFullImport(DataImporter.java:416)*
* at
org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:480)*
* at
org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:461)*
*Caused by: java.sql.SQLRecoverableException: No more data to read from
socket*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1200)*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1865)*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1757)*
* at
oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1750)*
* at
oracle.jdbc.driver.T4CClobAccessor.handlePrefetch(T4CClobAccessor.java:543)*
* at
oracle.jdbc.driver.T4CClobAccessor.unmarshalOneRow(T4CClobAccessor.java:197)*
* at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:916)*
* at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:835)*
* at oracle.jdbc.driver.T4C8Oall.readRXD(T4C8Oall.java:664)*
* at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:328)*
* at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:186)*
* at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:521)*
* at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:194)*
* at oracle.jdbc.driver.T4CStatement.fetch(T4CStatement.java:1074)*
* at
oracle.jdbc.driver.OracleResultSetImpl.close_or_fetch_from_next(OracleResultSetImpl.java:369)*
* at
oracle.jdbc.driver.OracleResultSetImpl.next(OracleResultSetImpl.java:273)*
* at
org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:370)*
* ... 12 more*
*db-data-config.xml:*
<dataConfig>
<script><![CDATA[
function makePair(row) {
var theKey = row.get("theKey")+ "_s";
var theValue = row.get("theValue");
row.put(theKey, theValue);
row.remove("theKey");
row.remove("theValue");
return row;
}
]]></script>
<dataSource name="a1" driver="oracle.jdbc.driver.OracleDriver"
url="***" user="***" password="***"/>
<dataSource name="a2" type="FieldReaderDataSource" encoding="UTF-8"/>
<document>
<entity name="e1" query="SELECT col1,col2, col3,col4,
XMLSERIALIZE(col5 AS CLOB) AS col5 FROM tableName"
dataSource="a1" pk="col1">
<field column="col1" name="col1" />
<field column="col2" name="col2" />
<field column="col3" name="col3" />
<field column="col4" name="col4" />
<entity name="e2" dataSource="a2"
processor="XPathEntityProcessor"
dataField="e1.col5"
forEach="/options/option"
transformer="script:makePair">
<field column="theKey" xpath="/options/option/@name" />
<field column="theValue" xpath="/options/option/value" />
</entity>
</entity>
</document>
</dataConfig>
Please Help me to resolve this issue.
Thanks,
Pankaj
Re: Solr limiting number of rows to indexed to 21500 every time.
Posted by Michael Della Bitta <mi...@appinions.com>.
Looks like you have an underlying JDBC problem. The socket representing
your database connection seems to be going away. Have you tried running
this query outside of Solr and iterating through all the results? How about
in a standalone Java program? Do you have a DBA you can consult to see if
there are any errors on the Oracle side?
Michael Della Bitta
Senior Software Engineer
o: +1 646 532 3062
appinions inc.
“The Science of Influence Marketing”
18 East 41st Street
New York, NY 10017
t: @appinions <https://twitter.com/Appinions> | g+:
plus.google.com/appinions
<https://plus.google.com/u/0/b/112002776285509593336/112002776285509593336/posts>
w: appinions.com <http://www.appinions.com/>
On Tue, Jan 13, 2015 at 2:31 AM, Pankaj Sonawane <pa...@gmail.com>
wrote:
> Hi,
>
> I am using Solr DataImportHandler to index data from database
> table(Oracle). One of the column contains String representation of XML
> (Sample below).
>
> *<options>*
> *<option name="A">1</option>*
>
> *<option name="B">2</option>*
>
> *<option name="C">3</option>*
> *.*
> *.*
> *.*
>
> *</options> //<option> can be 100-200*
>
> I want solr to index each 'name' in 'option' tag against its value
>
> ex. JSON for 1 row
> "docs": [ {
> "COL1": "F",
> "COL2": "ASDF", "COL3": "ATCC", "COL4": 29039757, "A_s": "1", "B_s": "2", "
> C_s": "3",
> .
> .
> .
> * }*
> // appending '_s' to 'name' attribute for making dynamic fields.
>
>
> But while indexing data, *every time only 21500 rows get indexed*. After
> these much records get indexed I got following exception:
>
> *1320927 [Thread-15] ERROR
> org.apache.solr.handler.dataimport.EntityProcessorBase û getNext() failed
> for query 'SELECT col1,col2,col3,col4,XMLSERIALIZE(col5 AS CLOB) AS col5
> FROM
> tableName':org.apache.solr.handler.dataimport.DataImportHandlerException:
> java.sql.SQLRecoverableException: No more data to read from socket*
> * at
>
> org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow(DataImportHandlerException.java:63)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:378)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.access$600(JdbcDataSource.java:258)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator$1.hasNext(JdbcDataSource.java:293)*
> * at
>
> org.apache.solr.handler.dataimport.EntityProcessorBase.getNext(EntityProcessorBase.java:116)*
> * at
>
> org.apache.solr.handler.dataimport.SqlEntityProcessor.nextRow(SqlEntityProcessor.java:75)*
> * at
>
> org.apache.solr.handler.dataimport.EntityProcessorWrapper.nextRow(EntityProcessorWrapper.java:243)*
> * at
>
> org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:476)*
> * at
>
> org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:415)*
> * at
>
> org.apache.solr.handler.dataimport.DocBuilder.doFullDump(DocBuilder.java:330)*
> * at
> org.apache.solr.handler.dataimport.DocBuilder.execute(DocBuilder.java:232)*
> * at
>
> org.apache.solr.handler.dataimport.DataImporter.doFullImport(DataImporter.java:416)*
> * at
>
> org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:480)*
> * at
>
> org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:461)*
> *Caused by: java.sql.SQLRecoverableException: No more data to read from
> socket*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1200)*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1865)*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1757)*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1750)*
> * at
>
> oracle.jdbc.driver.T4CClobAccessor.handlePrefetch(T4CClobAccessor.java:543)*
> * at
>
> oracle.jdbc.driver.T4CClobAccessor.unmarshalOneRow(T4CClobAccessor.java:197)*
> * at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:916)*
> * at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:835)*
> * at oracle.jdbc.driver.T4C8Oall.readRXD(T4C8Oall.java:664)*
> * at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:328)*
> * at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:186)*
> * at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:521)*
> * at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:194)*
> * at oracle.jdbc.driver.T4CStatement.fetch(T4CStatement.java:1074)*
> * at
>
> oracle.jdbc.driver.OracleResultSetImpl.close_or_fetch_from_next(OracleResultSetImpl.java:369)*
> * at
> oracle.jdbc.driver.OracleResultSetImpl.next(OracleResultSetImpl.java:273)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:370)*
> * ... 12 more*
>
> *1320928 [Thread-15] ERROR org.apache.solr.handler.dataimport.DocBuilder û
> Exception while processing: e1 document : SolrInputDocument(fields:
> []):org.apache.solr.handler.dataimport.DataImportHandlerException:
> java.sql.SQLRecoverableException: No more data to read from socket*
> * at
>
> org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow(DataImportHandlerException.java:63)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:378)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.access$600(JdbcDataSource.java:258)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator$1.hasNext(JdbcDataSource.java:293)*
> * at
>
> org.apache.solr.handler.dataimport.EntityProcessorBase.getNext(EntityProcessorBase.java:116)*
> * at
>
> org.apache.solr.handler.dataimport.SqlEntityProcessor.nextRow(SqlEntityProcessor.java:75)*
> * at
>
> org.apache.solr.handler.dataimport.EntityProcessorWrapper.nextRow(EntityProcessorWrapper.java:243)*
> * at
>
> org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:476)*
> * at
>
> org.apache.solr.handler.dataimport.DocBuilder.buildDocument(DocBuilder.java:415)*
> * at
>
> org.apache.solr.handler.dataimport.DocBuilder.doFullDump(DocBuilder.java:330)*
> * at
> org.apache.solr.handler.dataimport.DocBuilder.execute(DocBuilder.java:232)*
> * at
>
> org.apache.solr.handler.dataimport.DataImporter.doFullImport(DataImporter.java:416)*
> * at
>
> org.apache.solr.handler.dataimport.DataImporter.runCmd(DataImporter.java:480)*
> * at
>
> org.apache.solr.handler.dataimport.DataImporter$1.run(DataImporter.java:461)*
> *Caused by: java.sql.SQLRecoverableException: No more data to read from
> socket*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1200)*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1865)*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1757)*
> * at
> oracle.jdbc.driver.T4CMAREngine.unmarshalCLR(T4CMAREngine.java:1750)*
> * at
>
> oracle.jdbc.driver.T4CClobAccessor.handlePrefetch(T4CClobAccessor.java:543)*
> * at
>
> oracle.jdbc.driver.T4CClobAccessor.unmarshalOneRow(T4CClobAccessor.java:197)*
> * at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:916)*
> * at oracle.jdbc.driver.T4CTTIrxd.unmarshal(T4CTTIrxd.java:835)*
> * at oracle.jdbc.driver.T4C8Oall.readRXD(T4C8Oall.java:664)*
> * at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:328)*
> * at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:186)*
> * at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:521)*
> * at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:194)*
> * at oracle.jdbc.driver.T4CStatement.fetch(T4CStatement.java:1074)*
> * at
>
> oracle.jdbc.driver.OracleResultSetImpl.close_or_fetch_from_next(OracleResultSetImpl.java:369)*
> * at
> oracle.jdbc.driver.OracleResultSetImpl.next(OracleResultSetImpl.java:273)*
> * at
>
> org.apache.solr.handler.dataimport.JdbcDataSource$ResultSetIterator.hasnext(JdbcDataSource.java:370)*
> * ... 12 more*
>
>
> *db-data-config.xml:*
>
> <dataConfig>
> <script><![CDATA[
> function makePair(row) {
> var theKey = row.get("theKey")+ "_s";
> var theValue = row.get("theValue");
> row.put(theKey, theValue);
> row.remove("theKey");
> row.remove("theValue");
> return row;
> }
> ]]></script>
>
> <dataSource name="a1" driver="oracle.jdbc.driver.OracleDriver"
> url="***" user="***" password="***"/>
> <dataSource name="a2" type="FieldReaderDataSource" encoding="UTF-8"/>
>
> <document>
> <entity name="e1" query="SELECT col1,col2, col3,col4,
> XMLSERIALIZE(col5 AS CLOB) AS col5 FROM tableName"
> dataSource="a1" pk="col1">
> <field column="col1" name="col1" />
> <field column="col2" name="col2" />
> <field column="col3" name="col3" />
> <field column="col4" name="col4" />
> <entity name="e2" dataSource="a2"
> processor="XPathEntityProcessor"
> dataField="e1.col5"
> forEach="/options/option"
> transformer="script:makePair">
> <field column="theKey" xpath="/options/option/@name" />
> <field column="theValue" xpath="/options/option/value" />
> </entity>
> </entity>
> </document>
> </dataConfig>
>
> Please Help me to resolve this issue.
>
> Thanks,
> Pankaj
>