You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pig.apache.org by "liyunzhang_intel (JIRA)" <ji...@apache.org> on 2014/09/17 05:15:34 UTC
[jira] [Updated] (PIG-4183) fails run pig on spark and throw out
"java.lang.NoSuchMethodError: org.slf4j.spi.LocationAwareLogger.log"
[ https://issues.apache.org/jira/browse/PIG-4183?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
liyunzhang_intel updated PIG-4183:
----------------------------------
Status: Patch Available (was: Open)
The problem is because of using hadoop jar xxx to start pig.
the classpath of hadoop is like:
/home/zly/prj/oss/hadoop-1.2.1/conf:/usr/java/jdk1.7.0_51//lib/tools.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/..:/home/zly/prj/oss/hadoop-1.2.1/libexec/../hadoop-core-1.2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/asm-3.2.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/aspectjrt-1.6.11.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/aspectjtools-1.6.11.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-beanutils-1.7.0.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-beanutils-core-1.8.0.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-cli-1.2.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-codec-1.4.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-collections-3.2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-configuration-1.6.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-daemon-1.0.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-digester-1.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-el-1.0.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-httpclient-3.0.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-io-2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-lang-2.4.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-logging-1.1.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-logging-api-1.0.4.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-math-2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/commons-net-3.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/core-3.1.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/hadoop-capacity-scheduler-1.2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/hadoop-fairscheduler-1.2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/hadoop-thriftfs-1.2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/hsqldb-1.8.0.10.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jackson-core-asl-1.8.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jackson-mapper-asl-1.8.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jasper-compiler-5.5.12.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jasper-runtime-5.5.12.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jdeb-0.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jersey-core-1.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jersey-json-1.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jersey-server-1.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jets3t-0.6.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jetty-6.1.26.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jetty-util-6.1.26.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jsch-0.1.42.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/junit-4.5.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/kfs-0.2.2.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/log4j-1.2.15.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/mockito-all-1.8.5.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/oro-2.0.8.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/servlet-api-2.5-20081211.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/slf4j-api-1.4.3.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/slf4j-log4j12-1.4.3.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/xmlenc-0.52.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jsp-2.1/jsp-2.1.jar:/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/jsp-2.1/jsp-api-2.1.jar:/home/zly/prj/oss/pig/conf:/usr/java/jdk1.7.0_51/lib/tools.jar:/home/zly/prj/oss/pig/build/ivy/lib/Pig/*:/home/zly/prj/oss/hadoop-1.2.1/conf:/home/zly/prj/oss/hadoop-1.2.1/conf:/home/zly/prj/oss/pig/lib/ST4-4.0.4.jar:/home/zly/prj/oss/pig/lib/accumulo-core-1.5.0.jar:/home/zly/prj/oss/pig/lib/accumulo-fate-1.5.0.jar:/home/zly/prj/oss/pig/lib/accumulo-server-1.5.0.jar:/home/zly/prj/oss/pig/lib/accumulo-start-1.5.0.jar:/home/zly/prj/oss/pig/lib/accumulo-trace-1.5.0.jar:/home/zly/prj/oss/pig/lib/antlr-runtime-3.4.jar:/home/zly/prj/oss/pig/lib/asm-4.0.jar:/home/zly/prj/oss/pig/lib/asm-commons-4.0.jar:/home/zly/prj/oss/pig/lib/asm-tree-4.0.jar:/home/zly/prj/oss/pig/lib/automaton-1.11-8.jar:/home/zly/prj/oss/pig/lib/avro-1.7.5.jar:/home/zly/prj/oss/pig/lib/avro-tools-1.7.5-nodeps.jar:/home/zly/prj/oss/pig/lib/groovy-all-1.8.6.jar:/home/zly/prj/oss/pig/lib/guava-14.0.1.jar:/home/zly/prj/oss/pig/lib/hive-common-0.14.0-SNAPSHOT.jar:/home/zly/prj/oss/pig/lib/hive-exec-0.14.0-SNAPSHOT-core.jar:/home/zly/prj/oss/pig/lib/hive-serde-0.14.0-SNAPSHOT.jar:/home/zly/prj/oss/pig/lib/hive-shims-common-0.14.0-SNAPSHOT.jar:/home/zly/prj/oss/pig/lib/hive-shims-common-secure-0.14.0-SNAPSHOT.jar:/home/zly/prj/oss/pig/lib/jackson-core-asl-1.8.8.jar:/home/zly/prj/oss/pig/lib/jackson-mapper-asl-1.8.8.jar:/home/zly/prj/oss/pig/lib/jansi-1.9.jar:/home/zly/prj/oss/pig/lib/jline-1.0.jar:/home/zly/prj/oss/pig/lib/joda-time-2.1.jar:/home/zly/prj/oss/pig/lib/jruby-complete-1.6.7.jar:/home/zly/prj/oss/pig/lib/js-1.7R2.jar:/home/zly/prj/oss/pig/lib/json-simple-1.1.jar:/home/zly/prj/oss/pig/lib/jython-standalone-2.5.3.jar:/home/zly/prj/oss/pig/lib/protobuf-java-2.4.1-shaded.jar:/home/zly/prj/oss/pig/lib/protobuf-java-2.4.1.jar:/home/zly/prj/oss/pig/lib/protobuf-java-2.5.0.jar:/home/zly/prj/oss/pig/lib/snappy-java-1.0.5.jar:/home/zly/prj/oss/pig/lib/trevni-avro-1.7.5.jar:/home/zly/prj/oss/pig/lib/trevni-core-1.7.5.jar:/home/zly/prj/oss/pig/lib/zookeeper-3.4.5.jar:/home/zly/prj/oss/pig/pig-0.14.0-SNAPSHOT-core-h1.jar:/home/zly/prj/oss/pig/lib/h1/avro-mapred-1.7.5.jar:/home/zly/prj/oss/pig/lib/h1/hbase-client-0.96.0-hadoop1.jar:/home/zly/prj/oss/pig/lib/h1/hbase-common-0.96.0-hadoop1.jar:/home/zly/prj/oss/pig/lib/h1/hbase-hadoop-compat-0.96.0-hadoop1.jar:/home/zly/prj/oss/pig/lib/h1/hbase-hadoop1-compat-0.96.0-hadoop1.jar:/home/zly/prj/oss/pig/lib/h1/hbase-protocol-0.96.0-hadoop1.jar:/home/zly/prj/oss/pig/lib/h1/hbase-server-0.96.0-hadoop1.jar:/home/zly/prj/oss/pig/lib/h1/hive-shims-0.20S-0.14.0-SNAPSHOT.jar:
/home/zly/prj/oss/hadoop-1.2.1/libexec/../lib/slf4j-api-1.4.3.jar is loaded before slf4j-api-1.7.2.jar in /home/zly/prj/oss/pig/build/ivy/lib/Pig/, so slf4j jar conflictions are made.
The patch is made for adding hadoop_classpath(containing pig_classpath) before global classpath.
> fails run pig on spark and throw out "java.lang.NoSuchMethodError: org.slf4j.spi.LocationAwareLogger.log"
> ---------------------------------------------------------------------------------------------------------
>
> Key: PIG-4183
> URL: https://issues.apache.org/jira/browse/PIG-4183
> Project: Pig
> Issue Type: Bug
> Components: spark
> Reporter: liyunzhang_intel
> Assignee: liyunzhang_intel
>
> You can reproduce the bug in following steps
> 1.Build spark -0.9 env and build hadoop1 env:
> 2.Compile code: ant jar
> 3.Export $PIG_CLASSPATH
> echo $PIG_CLASSPATH
> /home/zly/prj/oss/pig/build/ivy/lib/Pig/*:/home/zly/prj/oss/hadoop-1.2.1/conf
> 4.Run: cd $PIG_HOME/bin; ./pig –x spark id.spark.pig
> 5.Error message found in pig log
> 549 ERROR 2998: Unhandled internal error. org.slf4j.spi.LocationAwareLogger.log(Lorg/slf4j/Marker;Ljava/lang/String;ILjava/lang/String;[Ljava/lang/Object;Ljava/lang/Throwable;)V
> 550
> 551 java.lang.NoSuchMethodError: org.slf4j.spi.LocationAwareLogger.log(Lorg/slf4j/Marker;Ljava/lang/String;ILjava/lang/String;[Ljava/lang/Object;Ljava/lang/Throwable;)V
> 552 at org.eclipse.jetty.util.log.JettyAwareLogger.log(JettyAwareLogger.java:607)
> 553 at org.eclipse.jetty.util.log.JettyAwareLogger.warn(JettyAwareLogger.java:431)
> 554 at org.eclipse.jetty.util.log.Slf4jLog.warn(Slf4jLog.java:69)
> 555 at org.eclipse.jetty.util.component.AbstractLifeCycle.setFailed(AbstractLifeCycle.java:204)
> 556 at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
> 557 at org.apache.spark.HttpServer.start(HttpServer.scala:65)
> 558 at org.apache.spark.broadcast.HttpBroadcast$.createServer(HttpBroadcast.scala:130)
> 559 at org.apache.spark.broadcast.HttpBroadcast$.initialize(HttpBroadcast.scala:101)
> 560 at org.apache.spark.broadcast.HttpBroadcastFactory.initialize(HttpBroadcast.scala:70)
> 561 at org.apache.spark.broadcast.BroadcastManager.initialize(Broadcast.scala:81)
> 562 at org.apache.spark.broadcast.BroadcastManager.<init>(Broadcast.scala:68)
> 563 at org.apache.spark.SparkEnv$.create(SparkEnv.scala:175)
> 564 at org.apache.spark.SparkContext.<init>(SparkContext.scala:139)
> 565 at org.apache.spark.SparkContext.<init>(SparkContext.scala:100)
> 566 at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:81)
> 567 at org.apache.pig.backend.hadoop.executionengine.spark.SparkLauncher.startSparkIfNeeded(SparkLauncher.java:202)
> 568 at org.apache.pig.backend.hadoop.executionengine.spark.SparkLauncher.launchPig(SparkLauncher.java:114)
> 569 at org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.launchPig(HExecutionEngine.java:279)
> 570 at org.apache.pig.PigServer.launchPlan(PigServer.java:1378)
> 571 at org.apache.pig.PigServer.executeCompiledLogicalPlan(PigServer.java:1363)
> 572 at org.apache.pig.PigServer.execute(PigServer.java:1352)
> 573 at org.apache.pig.PigServer.executeBatch(PigServer.java:403)
> 574 at org.apache.pig.PigServer.executeBatch(PigServer.java:386)
> 575 at org.apache.pig.tools.grunt.GruntParser.executeBatch(GruntParser.java:170)
> 576 at org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:233)
> 577 at org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:204)
> 578 at org.apache.pig.tools.grunt.Grunt.exec(Grunt.java:81)
> 579 at org.apache.pig.Main.run(Main.java:611)
> 580 at org.apache.pig.Main.main(Main.java:164)
> 581 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 582 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 583 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 584 at java.lang.reflect.Method.invoke(Method.java:606)
> 585 at org.apache.hadoop.util.RunJar.main(RunJar.java:160)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)