You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Michal Vince <vi...@gmail.com> on 2016/07/07 12:18:03 UTC
problem extracting map from json
Hi guys
I`m trying to extract Map[String, Any] from json string, this works well
in any scala repl I tried, both scala 2.11 and 2.10 and using both
json4s and liftweb-json libraries, but if I try to do the same thing in
spark-shell I`m always getting |No information known about type...|
exception
I`ved tried different versions of json4s and liftweb-json but with the
same result. I was wondering if anybody have idea what I might be doing
wrong.
I`m using spark 1.6.1 precompiled with mapr hadoop distro in scala 2.10.5
scala> import org.json4s._
import org.json4s._
scala> import org.json4s.jackson.JsonMethods._
import org.json4s.jackson.JsonMethods._
scala>
scala> implicit val formats = org.json4s.DefaultFormats
formats: org.json4s.DefaultFormats.type =
org.json4s.DefaultFormats$@46270641
scala> val json = parse(""" { "name": "joe", "children": [ { "name":
"Mary", "age": 5 }, { "name": "Mazy", "age": 3 } ] } """)
json: org.json4s.JValue = JObject(List((name,JString(joe)),
(children,JArray(List(JObject(List((name,JString(Mary)),
(age,JInt(5)))), JObject(List((name,JString(Mazy)), (age,JInt(3)))))))))
scala> json.extract[Map[String, Any]]
org.json4s.package$MappingException: No information known about type
at
org.json4s.Extraction$ClassInstanceBuilder.org$json4s$Extraction$ClassInstanceBuilder$$instantiate(Extraction.scala:465)
at
org.json4s.Extraction$ClassInstanceBuilder$$anonfun$result$6.apply(Extraction.scala:491)
at
org.json4s.Extraction$ClassInstanceBuilder$$anonfun$result$6.apply(Extraction.scala:488)
at
org.json4s.Extraction$.org$json4s$Extraction$$customOrElse(Extraction.scala:500)
at
org.json4s.Extraction$ClassInstanceBuilder.result(Extraction.scala:488)
at org.json4s.Extraction$.extract(Extraction.scala:332)
at
org.json4s.Extraction$$anonfun$extract$5.apply(Extraction.scala:316)
at
org.json4s.Extraction$$anonfun$extract$5.apply(Extraction.scala:316)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.immutable.List.foreach(List.scala:318)
at
scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.AbstractTraversable.map(Traversable.scala:105)
at org.json4s.Extraction$.extract(Extraction.scala:316)
at org.json4s.Extraction$.extract(Extraction.scala:42)
at
org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21)
at
$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:36)
at
$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:41)
at
$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:43)
at
$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:45)
at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:47)
at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:49)
at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:51)
at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:53)
at $iwC$$iwC$$iwC$$iwC.<init>(<console>:55)
at $iwC$$iwC$$iwC.<init>(<console>:57)
at $iwC$$iwC.<init>(<console>:59)
at $iwC.<init>(<console>:61)
at <init>(<console>:63)
at .<init>(<console>:67)
at .<clinit>(<console>)
at .<init>(<console>:7)
at .<clinit>(<console>)
at $print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)
at
org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1346)
at
org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)
at
org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:657)
at
org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:665)
at
org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$loop(SparkILoop.scala:670)
at
org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:997)
at
org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
at
org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
at
scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
at
org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)
at org.apache.spark.repl.Main$.main(Main.scala:31)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
best regards
Miso
Re: problem extracting map from json
Posted by Sivakumaran S <si...@me.com>.
Hi Michal,
Will an example help?
import scala.util.parsing.json._ //Requires scala-parsec-combinators because it is no longer part of core scala
val wbJSON = JSON.parseFull(weatherBox) //wbJSON is a JSON object now
//Depending on the structure, now traverse through the object
val listWeatherReports = wbJSON.get.asInstanceOf[Map[String, Any]]
val cod:String = listWeatherReports.get("cod").get.asInstanceOf[String]
Regards,
Sivakumaran
> On 07-Jul-2016, at 1:18 PM, Michal Vince <vi...@gmail.com> wrote:
>
> Hi guys
>
> I`m trying to extract Map[String, Any] from json string, this works well in any scala repl I tried, both scala 2.11 and 2.10 and using both json4s and liftweb-json libraries, but if I try to do the same thing in spark-shell I`m always getting No information known about type... exception
>
> I`ved tried different versions of json4s and liftweb-json but with the same result. I was wondering if anybody have idea what I might be doing wrong.
>
>
> I`m using spark 1.6.1 precompiled with mapr hadoop distro in scala 2.10.5
>
>
>
> scala> import org.json4s._
> import org.json4s._
>
> scala> import org.json4s.jackson.JsonMethods._
> import org.json4s.jackson.JsonMethods._
>
> scala>
>
> scala> implicit val formats = org.json4s.DefaultFormats
> formats: org.json4s.DefaultFormats.type = org.json4s.DefaultFormats$@46270641
>
> scala> val json = parse(""" { "name": "joe", "children": [ { "name": "Mary", "age": 5 }, { "name": "Mazy", "age": 3 } ] } """)
> json: org.json4s.JValue = JObject(List((name,JString(joe)), (children,JArray(List(JObject(List((name,JString(Mary)), (age,JInt(5)))), JObject(List((name,JString(Mazy)), (age,JInt(3)))))))))
>
>
> scala> json.extract[Map[String, Any]]
> org.json4s.package$MappingException: No information known about type
> at org.json4s.Extraction$ClassInstanceBuilder.org$json4s$Extraction$ClassInstanceBuilder$$instantiate(Extraction.scala:465)
> at org.json4s.Extraction$ClassInstanceBuilder$$anonfun$result$6.apply(Extraction.scala:491)
> at org.json4s.Extraction$ClassInstanceBuilder$$anonfun$result$6.apply(Extraction.scala:488)
> at org.json4s.Extraction$.org$json4s$Extraction$$customOrElse(Extraction.scala:500)
> at org.json4s.Extraction$ClassInstanceBuilder.result(Extraction.scala:488)
> at org.json4s.Extraction$.extract(Extraction.scala:332)
> at org.json4s.Extraction$$anonfun$extract$5.apply(Extraction.scala:316)
> at org.json4s.Extraction$$anonfun$extract$5.apply(Extraction.scala:316)
> at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
> at scala.collection.AbstractTraversable.map(Traversable.scala:105)
> at org.json4s.Extraction$.extract(Extraction.scala:316)
> at org.json4s.Extraction$.extract(Extraction.scala:42)
> at org.json4s.ExtractableJsonAstNode.extract(ExtractableJsonAstNode.scala:21)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:36)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:41)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:43)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:45)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:47)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:49)
> at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:51)
> at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:53)
> at $iwC$$iwC$$iwC$$iwC.<init>(<console>:55)
> at $iwC$$iwC$$iwC.<init>(<console>:57)
> at $iwC$$iwC.<init>(<console>:59)
> at $iwC.<init>(<console>:61)
> at <init>(<console>:63)
> at .<init>(<console>:67)
> at .<clinit>(<console>)
> at .<init>(<console>:7)
> at .<clinit>(<console>)
> at $print(<console>)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)
> at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1346)
> at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)
> at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)
> at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)
> at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
> at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
> at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
> at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:657)
> at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:665)
> at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$loop(SparkILoop.scala:670)
> at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:997)
> at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
> at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
> at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
> at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
> at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)
> at org.apache.spark.repl.Main$.main(Main.scala:31)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> best regards
>
> Miso