Spark flume java.lang.ArrayIndexOutOfBoundsException: 1

[centerqi hu <ce...@gmail.com>]

Hi all

This is a bug  of spark?


   val Array(host, IntParam(port)) = args


    val batchInterval = Milliseconds(120000)


    // Create the context and set the batch size

    val sparkConf = new SparkConf().setAppName("FlumeEventCount")

    val ssc = new StreamingContext(sparkConf, batchInterval)


    // Create a flume stream

    val stream = FlumeUtils.createStream(ssc, host, port,
StorageLevel.MEMORY_ONLY_SER_2)

    stream.map(event=>"Event: header:"+ event.event.get(0).toString+"
body:"+ new String(event.event.getBody.array) ).print



java.lang.ArrayIndexOutOfBoundsException: 1
        FlumeEventSaveHdfs$$anonfun$1$$anonfun$3.apply(FlumeEventCount.scala:62)
        FlumeEventSaveHdfs$$anonfun$1$$anonfun$3.apply(FlumeEventCount.scala:62)
        scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
        scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
        scala.collection.immutable.List.foreach(List.scala:318)
        scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
        scala.collection.AbstractTraversable.map(Traversable.scala:105)
        FlumeEventSaveHdfs$$anonfun$1.apply(FlumeEventCount.scala:62)
        FlumeEventSaveHdfs$$anonfun$1.apply(FlumeEventCount.scala:58)
        scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
        scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
        org.apache.spark.rdd.PairRDDFunctions$$anonfun$13.apply(PairRDDFunctions.scala:984)
        org.apache.spark.rdd.PairRDDFunctions$$anonfun$13.apply(PairRDDFunctions.scala:974)
        org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:62)
        org.apache.spark.scheduler.Task.run(Task.scala:54)
        org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:177)
        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

    ERROR scheduler.JobScheduler: Error running job streaming job
1411388760000 ms.0
org.apache.spark.SparkException: Job aborted due to stage failure:
Task 0 in stage 3.0 failed 4 times, most recent failure: Lost task 0.3
in stage 3.0 (TID 107, idc02-sp-sr-05.dns.abc.com):
java.lang.ArrayIndexOutOfBoundsException:

Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1185)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1174)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1173)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1173)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:688)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:688)
at scala.Option.foreach(Option.scala:236)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:688)
at org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1391)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
at akka.actor.ActorCell.invoke(ActorCell.scala:456)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
at akka.dispatch.Mailbox.run(Mailbox.scala:219)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

-- 
centerqi@gmail.com|齐忠

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@spark.apache.org
For additional commands, e-mail: user-help@spark.apache.org