You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@nutch.apache.org by Jurian Broertjes <ju...@openindex.io> on 2017/04/24 11:25:07 UTC
indexer-elastic version bump runtime dep issue
Hi list,
I'm trying to bump the ES lib version of the indexer-elastic plugin from
2.x to 5.x (in this case 5.3), but am running into runtime dependency
issues. What is the correct way of handling these kinds of errors? I'm
running Nutch 1.13
Exception:
java.lang.Exception: java.lang.NoSuchMethodError:
io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
at
org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
at
org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
Caused by: java.lang.NoSuchMethodError:
io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
at
org.elasticsearch.transport.netty4.Netty4Utils.toByteBuf(Netty4Utils.java:78)
at
org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:422)
at
org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:93)
at
org.elasticsearch.transport.TcpTransport.internalSendMessage(TcpTransport.java:1058)
at
org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1040)
at
org.elasticsearch.transport.TcpTransport.executeHandshake(TcpTransport.java:1555)
at
org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:502)
at
org.elasticsearch.transport.TcpTransport.connectToNode(TcpTransport.java:460)
at
org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:314)
at
org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler.doSample(TransportClientNodesService.java:408)
at
org.elasticsearch.client.transport.TransportClientNodesService$NodeSampler.sample(TransportClientNodesService.java:354)
at
org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddresses(TransportClientNodesService.java:195)
at
org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:322)
at
org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.makeClient(ElasticIndexWriter.java:170)
at
org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.open(ElasticIndexWriter.java:120)
at org.apache.nutch.indexer.IndexWriters.open(IndexWriters.java:77)
at
org.apache.nutch.indexer.IndexerOutputFormat.getRecordWriter(IndexerOutputFormat.java:39)
at
org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter.<init>(ReduceTask.java:484)
at
org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:414)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392)
at
org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Relevant parts:
ivy.xml:
<dependencies>
<dependency org="org.elasticsearch" name="elasticsearch"
rev="5.3.0" conf="*->default"/>
<dependency org="org.elasticsearch.client" name="transport"
rev="5.3.0"/>
<dependency org="org.apache.logging.log4j" name="log4j-api"
rev="2.7" />
<dependency org="org.apache.logging.log4j" name="log4j-core"
rev="2.7" />
<dependency org="io.netty" name="netty-all" rev="4.1.7.Final"/>
</dependencies>
plugin.xml:
<runtime>
<library name="indexer-elastic.jar">
<export name="*" />
</library>
<library name="rest-5.3.0.jar"/>
<library name="commons-codec-1.10.jar"/>
<library name="commons-logging-1.1.3.jar"/>
<library name="httpasyncclient-4.1.2.jar"/>
<library name="httpclient-4.5.2.jar"/>
<library name="httpcore-4.4.5.jar"/>
<library name="httpcore-nio-4.4.5.jar"/>
<library name="elasticsearch-5.3.0.jar"/>
<library name="compiler-0.9.3.jar"/>
<library name="HdrHistogram-2.1.6.jar"/>
<library name="hppc-0.7.1.jar"/>
<library name="jackson-core-2.8.6.jar"/>
<library name="jackson-dataformat-cbor-2.8.6.jar"/>
<library name="jackson-dataformat-smile-2.8.6.jar"/>
<library name="jackson-dataformat-yaml-2.8.6.jar"/>
<library name="jna-4.2.2.jar"/>
<library name="joda-time-2.9.5.jar"/>
<library name="jopt-simple-5.0.2.jar"/>
<library name="log4j-api-2.7.jar"/>
<library name="log4j-core-2.7.jar"/>
<library name="lucene-analyzers-common-6.4.1.jar"/>
<library name="lucene-backward-codecs-6.4.1.jar"/>
<library name="lucene-core-6.4.1.jar"/>
<library name="lucene-grouping-6.4.1.jar"/>
<library name="lucene-highlighter-6.4.1.jar"/>
<library name="lucene-join-6.4.1.jar"/>
<library name="lucene-memory-6.4.1.jar"/>
<library name="lucene-misc-6.4.1.jar"/>
<library name="lucene-queries-6.4.1.jar"/>
<library name="lucene-queryparser-6.4.1.jar"/>
<library name="lucene-sandbox-6.4.1.jar"/>
<library name="lucene-spatial3d-6.4.1.jar"/>
<library name="lucene-spatial-6.4.1.jar"/>
<library name="lucene-spatial-extras-6.4.1.jar"/>
<library name="lucene-suggest-6.4.1.jar"/>
<library name="securesm-1.1.jar"/>
<library name="snakeyaml-1.15.jar"/>
<library name="t-digest-3.0.jar"/>
<library name="percolator-client-5.3.0.jar"/>
<library name="reindex-client-5.3.0.jar"/>
<library name="lang-mustache-client-5.3.0.jar"/>
<library name="transport-5.3.0.jar"/>
<library name="transport-netty3-client-5.3.0.jar"/>
<library name="transport-netty4-client-5.3.0.jar"/>
<library name="netty-all-4.1.7.Final.jar"/>
<library name="netty-3.10.6.Final.jar"/>
<library name="netty-buffer-4.1.7.Final.jar"/>
<library name="netty-codec-4.1.7.Final.jar"/>
<library name="netty-codec-http-4.1.7.Final.jar"/>
<library name="netty-common-4.1.7.Final.jar"/>
<library name="netty-handler-4.1.7.Final.jar"/>
<library name="netty-resolver-4.1.7.Final.jar"/>
<library name="netty-transport-4.1.7.Final.jar"/>
</runtime>
I think the problem might be caused by a conflicting version of netty
jars. HDFS seems to pull 4.0.23 in, but ES needs 4.1.7. What can be done
to alleviate this?
If more info or src is needed, please let me know.
Best regards,
Jurian
Re: indexer-elastic version bump runtime dep issue
Posted by Sebastian Nagel <wa...@googlemail.com>.
Hi Jurian,
thanks, that's great news! I'll have a look at your patch.
Best,
Sebastian
On 05/01/2017 05:56 PM, Jurian Broertjes wrote:
> Hi Sebastian,
>
> I've continued to struggle with this on several levels (both local and on Hadoop), and in the end
> tried to change the way the classloader works (changed from parent-first to child-first) and this
> fixed the issues for me.
>
> I've created an issue[1] + patch for it. Can you let me know what you think of it?
>
> Best regards,
> Jurian
>
> [1] https://issues.apache.org/jira/browse/NUTCH-2378
>
> On 24-04-17 17:37, Sebastian Nagel wrote:
>> Hi Jurjan,
>>
>> it's a known problem that plugin dependencies and main dependencies may cause conflicts,
>> e.g, https://issues.apache.org/jira/browse/NUTCH-2316.
>> Every plugin is run in it's own class loader inheriting from the main class loader
>> but that's only prevents from conflicts between plugins.
>>
>> It's hard to say how and whether this can be fixed easily.
>> I would first try adjust or exclude netty from the main ivy/ivy.xmx,
>> or also just remove it from runtime/local/lib/
>>
>> Best,
>> Sebastian
>>
>> On 04/24/2017 01:25 PM, Jurian Broertjes wrote:
>>> Hi list,
>>>
>>> I'm trying to bump the ES lib version of the indexer-elastic plugin from 2.x to 5.x (in this case
>>> 5.3), but am running into runtime dependency issues. What is the correct way of handling these kinds
>>> of errors? I'm running Nutch 1.13
>>>
>>> Exception:
>>> java.lang.Exception: java.lang.NoSuchMethodError:
>>> io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
>>>
>>> at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
>>> at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
>>> Caused by: java.lang.NoSuchMethodError:
>>> io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
>>>
>>> at org.elasticsearch.transport.netty4.Netty4Utils.toByteBuf(Netty4Utils.java:78)
>>> at org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:422)
>>> at org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:93)
>>> at org.elasticsearch.transport.TcpTransport.internalSendMessage(TcpTransport.java:1058)
>>> at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1040)
>>> at org.elasticsearch.transport.TcpTransport.executeHandshake(TcpTransport.java:1555)
>>> at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:502)
>>> at org.elasticsearch.transport.TcpTransport.connectToNode(TcpTransport.java:460)
>>> at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:314)
>>> at
>>> org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler.doSample(TransportClientNodesService.java:408)
>>>
>>>
>>> at
>>> org.elasticsearch.client.transport.TransportClientNodesService$NodeSampler.sample(TransportClientNodesService.java:354)
>>>
>>>
>>> at
>>> org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddresses(TransportClientNodesService.java:195)
>>>
>>>
>>> at
>>> org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:322)
>>> at
>>> org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.makeClient(ElasticIndexWriter.java:170)
>>> at org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.open(ElasticIndexWriter.java:120)
>>> at org.apache.nutch.indexer.IndexWriters.open(IndexWriters.java:77)
>>> at org.apache.nutch.indexer.IndexerOutputFormat.getRecordWriter(IndexerOutputFormat.java:39)
>>> at org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter.<init>(ReduceTask.java:484)
>>> at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:414)
>>> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392)
>>> at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
>>> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
>>> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>>> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>> at java.lang.Thread.run(Thread.java:745)
>>>
>>>
>>>
>>>
>>> Relevant parts:
>>> ivy.xml:
>>> <dependencies>
>>> <dependency org="org.elasticsearch" name="elasticsearch" rev="5.3.0" conf="*->default"/>
>>> <dependency org="org.elasticsearch.client" name="transport" rev="5.3.0"/>
>>> <dependency org="org.apache.logging.log4j" name="log4j-api" rev="2.7" />
>>> <dependency org="org.apache.logging.log4j" name="log4j-core" rev="2.7" />
>>> <dependency org="io.netty" name="netty-all" rev="4.1.7.Final"/>
>>> </dependencies>
>>>
>>> plugin.xml:
>>> <runtime>
>>> <library name="indexer-elastic.jar">
>>> <export name="*" />
>>> </library>
>>>
>>> <library name="rest-5.3.0.jar"/>
>>> <library name="commons-codec-1.10.jar"/>
>>> <library name="commons-logging-1.1.3.jar"/>
>>> <library name="httpasyncclient-4.1.2.jar"/>
>>> <library name="httpclient-4.5.2.jar"/>
>>> <library name="httpcore-4.4.5.jar"/>
>>> <library name="httpcore-nio-4.4.5.jar"/>
>>>
>>> <library name="elasticsearch-5.3.0.jar"/>
>>> <library name="compiler-0.9.3.jar"/>
>>> <library name="HdrHistogram-2.1.6.jar"/>
>>> <library name="hppc-0.7.1.jar"/>
>>> <library name="jackson-core-2.8.6.jar"/>
>>> <library name="jackson-dataformat-cbor-2.8.6.jar"/>
>>> <library name="jackson-dataformat-smile-2.8.6.jar"/>
>>> <library name="jackson-dataformat-yaml-2.8.6.jar"/>
>>> <library name="jna-4.2.2.jar"/>
>>> <library name="joda-time-2.9.5.jar"/>
>>> <library name="jopt-simple-5.0.2.jar"/>
>>> <library name="log4j-api-2.7.jar"/>
>>> <library name="log4j-core-2.7.jar"/>
>>> <library name="lucene-analyzers-common-6.4.1.jar"/>
>>> <library name="lucene-backward-codecs-6.4.1.jar"/>
>>> <library name="lucene-core-6.4.1.jar"/>
>>> <library name="lucene-grouping-6.4.1.jar"/>
>>> <library name="lucene-highlighter-6.4.1.jar"/>
>>> <library name="lucene-join-6.4.1.jar"/>
>>> <library name="lucene-memory-6.4.1.jar"/>
>>> <library name="lucene-misc-6.4.1.jar"/>
>>> <library name="lucene-queries-6.4.1.jar"/>
>>> <library name="lucene-queryparser-6.4.1.jar"/>
>>> <library name="lucene-sandbox-6.4.1.jar"/>
>>> <library name="lucene-spatial3d-6.4.1.jar"/>
>>> <library name="lucene-spatial-6.4.1.jar"/>
>>> <library name="lucene-spatial-extras-6.4.1.jar"/>
>>> <library name="lucene-suggest-6.4.1.jar"/>
>>> <library name="securesm-1.1.jar"/>
>>> <library name="snakeyaml-1.15.jar"/>
>>> <library name="t-digest-3.0.jar"/>
>>> <library name="percolator-client-5.3.0.jar"/>
>>> <library name="reindex-client-5.3.0.jar"/>
>>> <library name="lang-mustache-client-5.3.0.jar"/>
>>> <library name="transport-5.3.0.jar"/>
>>> <library name="transport-netty3-client-5.3.0.jar"/>
>>> <library name="transport-netty4-client-5.3.0.jar"/>
>>> <library name="netty-all-4.1.7.Final.jar"/>
>>> <library name="netty-3.10.6.Final.jar"/>
>>> <library name="netty-buffer-4.1.7.Final.jar"/>
>>> <library name="netty-codec-4.1.7.Final.jar"/>
>>> <library name="netty-codec-http-4.1.7.Final.jar"/>
>>> <library name="netty-common-4.1.7.Final.jar"/>
>>> <library name="netty-handler-4.1.7.Final.jar"/>
>>> <library name="netty-resolver-4.1.7.Final.jar"/>
>>> <library name="netty-transport-4.1.7.Final.jar"/>
>>> </runtime>
>>>
>>> I think the problem might be caused by a conflicting version of netty jars. HDFS seems to pull
>>> 4.0.23 in, but ES needs 4.1.7. What can be done to alleviate this?
>>>
>>> If more info or src is needed, please let me know.
>>>
>>> Best regards,
>>> Jurian
>
Re: indexer-elastic version bump runtime dep issue
Posted by Jurian Broertjes <ju...@openindex.io>.
Hi Sebastian,
I've continued to struggle with this on several levels (both local and
on Hadoop), and in the end tried to change the way the classloader works
(changed from parent-first to child-first) and this fixed the issues for
me.
I've created an issue[1] + patch for it. Can you let me know what you
think of it?
Best regards,
Jurian
[1] https://issues.apache.org/jira/browse/NUTCH-2378
On 24-04-17 17:37, Sebastian Nagel wrote:
> Hi Jurjan,
>
> it's a known problem that plugin dependencies and main dependencies may cause conflicts,
> e.g, https://issues.apache.org/jira/browse/NUTCH-2316.
> Every plugin is run in it's own class loader inheriting from the main class loader
> but that's only prevents from conflicts between plugins.
>
> It's hard to say how and whether this can be fixed easily.
> I would first try adjust or exclude netty from the main ivy/ivy.xmx,
> or also just remove it from runtime/local/lib/
>
> Best,
> Sebastian
>
> On 04/24/2017 01:25 PM, Jurian Broertjes wrote:
>> Hi list,
>>
>> I'm trying to bump the ES lib version of the indexer-elastic plugin from 2.x to 5.x (in this case
>> 5.3), but am running into runtime dependency issues. What is the correct way of handling these kinds
>> of errors? I'm running Nutch 1.13
>>
>> Exception:
>> java.lang.Exception: java.lang.NoSuchMethodError:
>> io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
>> at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
>> at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
>> Caused by: java.lang.NoSuchMethodError:
>> io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
>> at org.elasticsearch.transport.netty4.Netty4Utils.toByteBuf(Netty4Utils.java:78)
>> at org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:422)
>> at org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:93)
>> at org.elasticsearch.transport.TcpTransport.internalSendMessage(TcpTransport.java:1058)
>> at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1040)
>> at org.elasticsearch.transport.TcpTransport.executeHandshake(TcpTransport.java:1555)
>> at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:502)
>> at org.elasticsearch.transport.TcpTransport.connectToNode(TcpTransport.java:460)
>> at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:314)
>> at
>> org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler.doSample(TransportClientNodesService.java:408)
>>
>> at
>> org.elasticsearch.client.transport.TransportClientNodesService$NodeSampler.sample(TransportClientNodesService.java:354)
>>
>> at
>> org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddresses(TransportClientNodesService.java:195)
>>
>> at org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:322)
>> at org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.makeClient(ElasticIndexWriter.java:170)
>> at org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.open(ElasticIndexWriter.java:120)
>> at org.apache.nutch.indexer.IndexWriters.open(IndexWriters.java:77)
>> at org.apache.nutch.indexer.IndexerOutputFormat.getRecordWriter(IndexerOutputFormat.java:39)
>> at org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter.<init>(ReduceTask.java:484)
>> at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:414)
>> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392)
>> at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
>> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
>> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>>
>> Relevant parts:
>> ivy.xml:
>> <dependencies>
>> <dependency org="org.elasticsearch" name="elasticsearch" rev="5.3.0" conf="*->default"/>
>> <dependency org="org.elasticsearch.client" name="transport" rev="5.3.0"/>
>> <dependency org="org.apache.logging.log4j" name="log4j-api" rev="2.7" />
>> <dependency org="org.apache.logging.log4j" name="log4j-core" rev="2.7" />
>> <dependency org="io.netty" name="netty-all" rev="4.1.7.Final"/>
>> </dependencies>
>>
>> plugin.xml:
>> <runtime>
>> <library name="indexer-elastic.jar">
>> <export name="*" />
>> </library>
>>
>> <library name="rest-5.3.0.jar"/>
>> <library name="commons-codec-1.10.jar"/>
>> <library name="commons-logging-1.1.3.jar"/>
>> <library name="httpasyncclient-4.1.2.jar"/>
>> <library name="httpclient-4.5.2.jar"/>
>> <library name="httpcore-4.4.5.jar"/>
>> <library name="httpcore-nio-4.4.5.jar"/>
>>
>> <library name="elasticsearch-5.3.0.jar"/>
>> <library name="compiler-0.9.3.jar"/>
>> <library name="HdrHistogram-2.1.6.jar"/>
>> <library name="hppc-0.7.1.jar"/>
>> <library name="jackson-core-2.8.6.jar"/>
>> <library name="jackson-dataformat-cbor-2.8.6.jar"/>
>> <library name="jackson-dataformat-smile-2.8.6.jar"/>
>> <library name="jackson-dataformat-yaml-2.8.6.jar"/>
>> <library name="jna-4.2.2.jar"/>
>> <library name="joda-time-2.9.5.jar"/>
>> <library name="jopt-simple-5.0.2.jar"/>
>> <library name="log4j-api-2.7.jar"/>
>> <library name="log4j-core-2.7.jar"/>
>> <library name="lucene-analyzers-common-6.4.1.jar"/>
>> <library name="lucene-backward-codecs-6.4.1.jar"/>
>> <library name="lucene-core-6.4.1.jar"/>
>> <library name="lucene-grouping-6.4.1.jar"/>
>> <library name="lucene-highlighter-6.4.1.jar"/>
>> <library name="lucene-join-6.4.1.jar"/>
>> <library name="lucene-memory-6.4.1.jar"/>
>> <library name="lucene-misc-6.4.1.jar"/>
>> <library name="lucene-queries-6.4.1.jar"/>
>> <library name="lucene-queryparser-6.4.1.jar"/>
>> <library name="lucene-sandbox-6.4.1.jar"/>
>> <library name="lucene-spatial3d-6.4.1.jar"/>
>> <library name="lucene-spatial-6.4.1.jar"/>
>> <library name="lucene-spatial-extras-6.4.1.jar"/>
>> <library name="lucene-suggest-6.4.1.jar"/>
>> <library name="securesm-1.1.jar"/>
>> <library name="snakeyaml-1.15.jar"/>
>> <library name="t-digest-3.0.jar"/>
>> <library name="percolator-client-5.3.0.jar"/>
>> <library name="reindex-client-5.3.0.jar"/>
>> <library name="lang-mustache-client-5.3.0.jar"/>
>> <library name="transport-5.3.0.jar"/>
>> <library name="transport-netty3-client-5.3.0.jar"/>
>> <library name="transport-netty4-client-5.3.0.jar"/>
>> <library name="netty-all-4.1.7.Final.jar"/>
>> <library name="netty-3.10.6.Final.jar"/>
>> <library name="netty-buffer-4.1.7.Final.jar"/>
>> <library name="netty-codec-4.1.7.Final.jar"/>
>> <library name="netty-codec-http-4.1.7.Final.jar"/>
>> <library name="netty-common-4.1.7.Final.jar"/>
>> <library name="netty-handler-4.1.7.Final.jar"/>
>> <library name="netty-resolver-4.1.7.Final.jar"/>
>> <library name="netty-transport-4.1.7.Final.jar"/>
>> </runtime>
>>
>> I think the problem might be caused by a conflicting version of netty jars. HDFS seems to pull
>> 4.0.23 in, but ES needs 4.1.7. What can be done to alleviate this?
>>
>> If more info or src is needed, please let me know.
>>
>> Best regards,
>> Jurian
Re: indexer-elastic version bump runtime dep issue
Posted by Sebastian Nagel <wa...@googlemail.com>.
Hi Jurjan,
it's a known problem that plugin dependencies and main dependencies may cause conflicts,
e.g, https://issues.apache.org/jira/browse/NUTCH-2316.
Every plugin is run in it's own class loader inheriting from the main class loader
but that's only prevents from conflicts between plugins.
It's hard to say how and whether this can be fixed easily.
I would first try adjust or exclude netty from the main ivy/ivy.xmx,
or also just remove it from runtime/local/lib/
Best,
Sebastian
On 04/24/2017 01:25 PM, Jurian Broertjes wrote:
> Hi list,
>
> I'm trying to bump the ES lib version of the indexer-elastic plugin from 2.x to 5.x (in this case
> 5.3), but am running into runtime dependency issues. What is the correct way of handling these kinds
> of errors? I'm running Nutch 1.13
>
> Exception:
> java.lang.Exception: java.lang.NoSuchMethodError:
> io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
> at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
> at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
> Caused by: java.lang.NoSuchMethodError:
> io.netty.buffer.CompositeByteBuf.addComponents(ZLjava/lang/Iterable;)Lio/netty/buffer/CompositeByteBuf;
> at org.elasticsearch.transport.netty4.Netty4Utils.toByteBuf(Netty4Utils.java:78)
> at org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:422)
> at org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:93)
> at org.elasticsearch.transport.TcpTransport.internalSendMessage(TcpTransport.java:1058)
> at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1040)
> at org.elasticsearch.transport.TcpTransport.executeHandshake(TcpTransport.java:1555)
> at org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:502)
> at org.elasticsearch.transport.TcpTransport.connectToNode(TcpTransport.java:460)
> at org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:314)
> at
> org.elasticsearch.client.transport.TransportClientNodesService$SimpleNodeSampler.doSample(TransportClientNodesService.java:408)
>
> at
> org.elasticsearch.client.transport.TransportClientNodesService$NodeSampler.sample(TransportClientNodesService.java:354)
>
> at
> org.elasticsearch.client.transport.TransportClientNodesService.addTransportAddresses(TransportClientNodesService.java:195)
>
> at org.elasticsearch.client.transport.TransportClient.addTransportAddress(TransportClient.java:322)
> at org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.makeClient(ElasticIndexWriter.java:170)
> at org.apache.nutch.indexwriter.elastic.ElasticIndexWriter.open(ElasticIndexWriter.java:120)
> at org.apache.nutch.indexer.IndexWriters.open(IndexWriters.java:77)
> at org.apache.nutch.indexer.IndexerOutputFormat.getRecordWriter(IndexerOutputFormat.java:39)
> at org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter.<init>(ReduceTask.java:484)
> at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:414)
> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392)
> at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Relevant parts:
> ivy.xml:
> <dependencies>
> <dependency org="org.elasticsearch" name="elasticsearch" rev="5.3.0" conf="*->default"/>
> <dependency org="org.elasticsearch.client" name="transport" rev="5.3.0"/>
> <dependency org="org.apache.logging.log4j" name="log4j-api" rev="2.7" />
> <dependency org="org.apache.logging.log4j" name="log4j-core" rev="2.7" />
> <dependency org="io.netty" name="netty-all" rev="4.1.7.Final"/>
> </dependencies>
>
> plugin.xml:
> <runtime>
> <library name="indexer-elastic.jar">
> <export name="*" />
> </library>
>
> <library name="rest-5.3.0.jar"/>
> <library name="commons-codec-1.10.jar"/>
> <library name="commons-logging-1.1.3.jar"/>
> <library name="httpasyncclient-4.1.2.jar"/>
> <library name="httpclient-4.5.2.jar"/>
> <library name="httpcore-4.4.5.jar"/>
> <library name="httpcore-nio-4.4.5.jar"/>
>
> <library name="elasticsearch-5.3.0.jar"/>
> <library name="compiler-0.9.3.jar"/>
> <library name="HdrHistogram-2.1.6.jar"/>
> <library name="hppc-0.7.1.jar"/>
> <library name="jackson-core-2.8.6.jar"/>
> <library name="jackson-dataformat-cbor-2.8.6.jar"/>
> <library name="jackson-dataformat-smile-2.8.6.jar"/>
> <library name="jackson-dataformat-yaml-2.8.6.jar"/>
> <library name="jna-4.2.2.jar"/>
> <library name="joda-time-2.9.5.jar"/>
> <library name="jopt-simple-5.0.2.jar"/>
> <library name="log4j-api-2.7.jar"/>
> <library name="log4j-core-2.7.jar"/>
> <library name="lucene-analyzers-common-6.4.1.jar"/>
> <library name="lucene-backward-codecs-6.4.1.jar"/>
> <library name="lucene-core-6.4.1.jar"/>
> <library name="lucene-grouping-6.4.1.jar"/>
> <library name="lucene-highlighter-6.4.1.jar"/>
> <library name="lucene-join-6.4.1.jar"/>
> <library name="lucene-memory-6.4.1.jar"/>
> <library name="lucene-misc-6.4.1.jar"/>
> <library name="lucene-queries-6.4.1.jar"/>
> <library name="lucene-queryparser-6.4.1.jar"/>
> <library name="lucene-sandbox-6.4.1.jar"/>
> <library name="lucene-spatial3d-6.4.1.jar"/>
> <library name="lucene-spatial-6.4.1.jar"/>
> <library name="lucene-spatial-extras-6.4.1.jar"/>
> <library name="lucene-suggest-6.4.1.jar"/>
> <library name="securesm-1.1.jar"/>
> <library name="snakeyaml-1.15.jar"/>
> <library name="t-digest-3.0.jar"/>
> <library name="percolator-client-5.3.0.jar"/>
> <library name="reindex-client-5.3.0.jar"/>
> <library name="lang-mustache-client-5.3.0.jar"/>
> <library name="transport-5.3.0.jar"/>
> <library name="transport-netty3-client-5.3.0.jar"/>
> <library name="transport-netty4-client-5.3.0.jar"/>
> <library name="netty-all-4.1.7.Final.jar"/>
> <library name="netty-3.10.6.Final.jar"/>
> <library name="netty-buffer-4.1.7.Final.jar"/>
> <library name="netty-codec-4.1.7.Final.jar"/>
> <library name="netty-codec-http-4.1.7.Final.jar"/>
> <library name="netty-common-4.1.7.Final.jar"/>
> <library name="netty-handler-4.1.7.Final.jar"/>
> <library name="netty-resolver-4.1.7.Final.jar"/>
> <library name="netty-transport-4.1.7.Final.jar"/>
> </runtime>
>
> I think the problem might be caused by a conflicting version of netty jars. HDFS seems to pull
> 4.0.23 in, but ES needs 4.1.7. What can be done to alleviate this?
>
> If more info or src is needed, please let me know.
>
> Best regards,
> Jurian