You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flink.apache.org by Fanbin Bu <fa...@coinbase.com> on 2019/12/17 00:10:36 UTC
flink 1.9 conflict jackson version
Hi,
After I upgrade flink 1.9, I got the following error message on EMR, it
works locally on IntelliJ.
I'm explicitly declaring the dependency as
implementation
'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.1'
and I have
implementation group: 'com.amazonaws', name: 'aws-java-sdk-emr', version:
'1.11.595'
java.lang.NoSuchMethodError:
com.fasterxml.jackson.databind.ObjectMapper.enable([Lcom/fasterxml/jackson/core/JsonParser$Feature;)Lcom/fasterxml/jackson/databind/ObjectMapper;
at com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:54)
at com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:65)
at com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:53)
at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:107)
at com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:256)
at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:460)
at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424)
at com.amazonaws.client.builder.AwsAsyncClientBuilder.build(AwsAsyncClientBuilder.java:80)
at com.coinbase.util.KmsClient$.getSnowflakeUsernamePassword(KmsClient.scala:21)
at com.coinbase.ml.RunFlinkJob$.runBatch(RunFlinkJob.scala:94)
at com.coinbase.ml.RunFlinkJob$.runFlinkJob(RunFlinkJob.scala:38)
at com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint$.main(CmdLineParser.scala:76)
at com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint.main(CmdLineParser.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:576)
at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:438)
at org.apache.flink.client.program.ClusterClient.run(ClusterClient.java:274)
at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:746)
at org.apache.flink.client.cli.CliFrontend.runProgram(CliFrontend.java:273)
at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:205)
at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:1010)
at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1083)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1844)
at org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1083)
Re: flink 1.9 conflict jackson version
Posted by Fanbin Bu <fa...@coinbase.com>.
Hi Aj,
I got a work around to put my app jar inside /usr/lib/flink/lib directory.
On Mon, Apr 6, 2020 at 11:27 PM aj <aj...@gmail.com> wrote:
> Hi Fanbin,
>
> I am facing a similar kind of issue. Let me know if you are able to
> resolve this issue then please help me also
>
>
> https://stackoverflow.com/questions/61012350/flink-reading-a-s3-file-causing-jackson-dependency-issue
>
>
>
> On Tue, Dec 17, 2019 at 7:50 AM ouywl <ou...@139.com> wrote:
>
>> Hi Bu
>> I think It can use mvn-shade-plugin to resolve your problem, It seem
>> flink-client conflict with your owner jar?
>>
>> ouywl
>> ouywl@139.com
>>
>> <https://maas.mail.163.com/dashi-web-extend/html/proSignature.html?ftlId=1&name=ouywl&uid=ouywl%40139.com&iconUrl=https%3A%2F%2Fmail-online.nosdn.127.net%2Fsma8dc7719018ba2517da7111b3db5a170.jpg&items=%5B%22ouywl%40139.com%22%5D>
>> 签名由 网易邮箱大师 <https://mail.163.com/dashi/dlpro.html?from=mail81> 定制
>>
>> On 12/17/2019 08:10，Fanbin Bu<fa...@coinbase.com>
>> <fa...@coinbase.com> wrote：
>>
>> Hi,
>>
>> After I upgrade flink 1.9, I got the following error message on EMR, it
>> works locally on IntelliJ.
>>
>> I'm explicitly declaring the dependency as
>> implementation
>> 'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.1'
>> and I have
>> implementation group: 'com.amazonaws', name: 'aws-java-sdk-emr', version:
>> '1.11.595'
>>
>>
>>
>> java.lang.NoSuchMethodError: com.fasterxml.jackson.databind.ObjectMapper.enable([Lcom/fasterxml/jackson/core/JsonParser$Feature;)Lcom/fasterxml/jackson/databind/ObjectMapper;
>> at com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:54)
>> at com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
>> at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:65)
>> at com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:53)
>> at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:107)
>> at com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:256)
>> at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:460)
>> at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424)
>> at com.amazonaws.client.builder.AwsAsyncClientBuilder.build(AwsAsyncClientBuilder.java:80)
>> at com.coinbase.util.KmsClient$.getSnowflakeUsernamePassword(KmsClient.scala:21)
>> at com.coinbase.ml.RunFlinkJob$.runBatch(RunFlinkJob.scala:94)
>> at com.coinbase.ml.RunFlinkJob$.runFlinkJob(RunFlinkJob.scala:38)
>> at com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint$.main(CmdLineParser.scala:76)
>> at com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint.main(CmdLineParser.scala)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:498)
>> at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:576)
>> at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:438)
>> at org.apache.flink.client.program.ClusterClient.run(ClusterClient.java:274)
>> at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:746)
>> at org.apache.flink.client.cli.CliFrontend.runProgram(CliFrontend.java:273)
>> at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:205)
>> at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:1010)
>> at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1083)
>> at java.security.AccessController.doPrivileged(Native Method)
>> at javax.security.auth.Subject.doAs(Subject.java:422)
>> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1844)
>> at org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
>> at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1083)
>>
>>
>
> --
> Thanks & Regards,
> Anuj Jain
> Mob. : +91- 8588817877
> Skype : anuj.jain07
> <http://www.oracle.com/>
>
>
> <http://www.cse.iitm.ac.in/%7Eanujjain/>
>
Re: flink 1.9 conflict jackson version
Posted by aj <aj...@gmail.com>.
Hi Fanbin,
I am facing a similar kind of issue. Let me know if you are able to
resolve this issue then please help me also
https://stackoverflow.com/questions/61012350/flink-reading-a-s3-file-causing-jackson-dependency-issue
On Tue, Dec 17, 2019 at 7:50 AM ouywl <ou...@139.com> wrote:
> Hi Bu
> I think It can use mvn-shade-plugin to resolve your problem, It seem
> flink-client conflict with your owner jar?
>
> ouywl
> ouywl@139.com
>
> <https://maas.mail.163.com/dashi-web-extend/html/proSignature.html?ftlId=1&name=ouywl&uid=ouywl%40139.com&iconUrl=https%3A%2F%2Fmail-online.nosdn.127.net%2Fsma8dc7719018ba2517da7111b3db5a170.jpg&items=%5B%22ouywl%40139.com%22%5D>
> 签名由 网易邮箱大师 <https://mail.163.com/dashi/dlpro.html?from=mail81> 定制
>
> On 12/17/2019 08:10，Fanbin Bu<fa...@coinbase.com>
> <fa...@coinbase.com> wrote：
>
> Hi,
>
> After I upgrade flink 1.9, I got the following error message on EMR, it
> works locally on IntelliJ.
>
> I'm explicitly declaring the dependency as
> implementation
> 'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.1'
> and I have
> implementation group: 'com.amazonaws', name: 'aws-java-sdk-emr', version:
> '1.11.595'
>
>
>
> java.lang.NoSuchMethodError: com.fasterxml.jackson.databind.ObjectMapper.enable([Lcom/fasterxml/jackson/core/JsonParser$Feature;)Lcom/fasterxml/jackson/databind/ObjectMapper;
> at com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:54)
> at com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
> at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:65)
> at com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:53)
> at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:107)
> at com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:256)
> at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:460)
> at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424)
> at com.amazonaws.client.builder.AwsAsyncClientBuilder.build(AwsAsyncClientBuilder.java:80)
> at com.coinbase.util.KmsClient$.getSnowflakeUsernamePassword(KmsClient.scala:21)
> at com.coinbase.ml.RunFlinkJob$.runBatch(RunFlinkJob.scala:94)
> at com.coinbase.ml.RunFlinkJob$.runFlinkJob(RunFlinkJob.scala:38)
> at com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint$.main(CmdLineParser.scala:76)
> at com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint.main(CmdLineParser.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:576)
> at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:438)
> at org.apache.flink.client.program.ClusterClient.run(ClusterClient.java:274)
> at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:746)
> at org.apache.flink.client.cli.CliFrontend.runProgram(CliFrontend.java:273)
> at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:205)
> at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:1010)
> at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1083)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1844)
> at org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
> at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1083)
>
>
--
Thanks & Regards,
Anuj Jain
Mob. : +91- 8588817877
Skype : anuj.jain07
<http://www.oracle.com/>
<http://www.cse.iitm.ac.in/%7Eanujjain/>
Re:flink 1.9 conflict jackson version
Posted by ouywl <ou...@139.com>.
Hi Bu
I think It can use mvn-shade-plugin to resolve your problem, It seem
flink-client conflict with your owner jar?
[  | ouywl
---|---
ouywl@139.com
](https://maas.mail.163.com/dashi-web-
extend/html/proSignature.html?ftlId=1&name=ouywl&uid=ouywl%40139.com&iconUrl=https%3A%2F%2Fmail-
online.nosdn.127.net%2Fsma8dc7719018ba2517da7111b3db5a170.jpg&items=%5B%22ouywl%40139.com%22%5D)
签名由 [网易邮箱大师](https://mail.163.com/dashi/dlpro.html?from=mail81) 定制
On 12/17/2019 08:10，[Fanbin
Bu<fa...@coinbase.com>](mailto:fanbin.bu@coinbase.com) wrote：
> Hi,
>
>
>
>
>
> After I upgrade flink 1.9, I got the following error message on EMR, it
works locally on IntelliJ.
>
>
>
>
> I'm explicitly declaring the dependency as
>
> implementation 'com.fasterxml.jackson.module:jackson-module-
scala_2.11:2.10.1'
>
>
> and I have
>
> implementation group: 'com.amazonaws', name: 'aws-java-sdk-emr', version:
'1.11.595'
>
>
>
>
>
>
>
>
>
>
>
>
> java.lang.NoSuchMethodError:
com.fasterxml.jackson.databind.ObjectMapper.enable([Lcom/fasterxml/jackson/core/JsonParser$Feature;)Lcom/fasterxml/jackson/databind/ObjectMapper;
> at
com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:54)
> at
com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
> at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:65)
> at
com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:53)
> at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:107)
> at
com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:256)
> at
com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:460)
> at
com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424)
> at
com.amazonaws.client.builder.AwsAsyncClientBuilder.build(AwsAsyncClientBuilder.java:80)
> at
com.coinbase.util.KmsClient$.getSnowflakeUsernamePassword(KmsClient.scala:21)
> at com.coinbase.ml.RunFlinkJob$.runBatch(RunFlinkJob.scala:94)
> at com.coinbase.ml.RunFlinkJob$.runFlinkJob(RunFlinkJob.scala:38)
> at
com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint$.main(CmdLineParser.scala:76)
> at
com.coinbase.ml.FlinkFeatureProcessingJobEntryPoint.main(CmdLineParser.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:576)
> at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:438)
> at
org.apache.flink.client.program.ClusterClient.run(ClusterClient.java:274)
> at
org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:746)
> at
org.apache.flink.client.cli.CliFrontend.runProgram(CliFrontend.java:273)
> at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:205)
> at
org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:1010)
> at
org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1083)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1844)
> at
org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
> at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1083)