You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Hyukjin Kwon (JIRA)" <ji...@apache.org> on 2019/05/29 01:20:00 UTC
[jira] [Updated] (SPARK-27861) get_json_object in sql will truncate long value gotten from jsonpath

     [ https://issues.apache.org/jira/browse/SPARK-27861?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Hyukjin Kwon updated SPARK-27861:
---------------------------------
    Description: 
{code:java}
select get_json_object('{"key":"eyJib29sIjp7ImZpbHRlciI6W3sidGVybSI6eyJ3YXJlaG91c2VfaWQiOjMwfX0seyJ0ZXJtcyI6eyJpZCI6WzE2MTgzMjk3NywxNjE4MjM1OTgsMTYxODIzNTk3LDE2MTgyMzU5NiwxNjE4MjM1OTUsMTYxODAwMjI0LDE2MTgwMDIyMSwxNjE4MDAyMjAsMTYxODAwMjE5LDE2MTgwMDIxOCwxNjE4MDAyMTMsMTYxODAwMjEyLDE2MTgwMDIxMSwxNjE4MDAyMTAsMTYxODAwMjA2LDE2MTgwMDIwMiwxNjE4MDAyMDEsMTYxODAwMTk4LDE2MTgwMDE5NywxNjE4MDAxOTYsMTYxODAwMTk1LDE2MTgwMDE5NCwxNjE4MDAxOTMsMTYxODAwMTkyLDE2MTgwMDE5MSwxNjE4MDAxOTAsMTYxODAwMTg5LDE2MTc5NDAyOSwxNjEzNTg2ODMsMTYxMzU4NjExLDE2MTM1ODYxMCwxNjEzNTYzMTIsMTYxMzU0NzU5LDE2MTM1NDY5NywxNjEzNTQ2OTYsMTYxMzQ0NTkyLDE2MTM0NDUwNywxNjEzMzc2MzQsMTYxMzM3MjA2LDE2MTMwMDM4OCwxNjEzMDAzODcsMTYxMzAwMzg2LDE2MTI4ODAxMCwxNjEyODgwMDksMTYxMjc5NTI0LDE2MTI3OTUyMiwxNjEyNzk1MTUsMTYxMjc5NTE0LDE2MTI3OTUxMywxNjEyNzg5NzcsMTYxMjc4OTc2LDE2MTI3ODk0NCwxNjEyNzg5NDMsMTYwODk2NjM3LDE2MDg5NjYzNiwxNjA4NzQ0ODcsMTYwODc0NDg2LDE2MDg3NDQ2NywxNjA4NzQ0NjYsMTYwODc0NDY1LDE2MDg3NDQ2NCwxNjA4NzQ0NjMsMTYwODc0NDU2LDE2MDg3NDQ1NSwxNjA4NzQxNTksMTYwODc0MTU2LDE2MDg3NDE1NSwxNjA4NzQxNTIsMTYwODc0MTUxLDE2MDg3NDE1MCwxNjA4NzQxNDksMTYwODc0MTQ4LDE2MDg3NDE0NSwxNjA4NzQxNDQsMTYwODc0MTQzLDE2MDg3NDE0MiwxNjA4NzQxNDEsMTYwODc0MTQwLDE2MDg3NDEzNywxNjA4NzQxMzYsMTYwODc0MTMzLDE2MDg3NDEzMiwxNjA4MjI0OTksMTYwODExMDE5LDE2MDgxMDkxNSwxNjA4MTA5MTQsMTYwODEwNDU4LDE2MDgxMDQ1NywxNjA4MTA0NTYsMTYwODEwNDU1LDE2MDgxMDQ1NCwxNjA4MTAzNTMsMTYwNzk5NTU1LDE2MDc5MjQ1OSwxNjA3OTI0NTYsMTYwNzkyNDU1LDE2MDc5MjQ1NCwxNjA3OTI0NTEsMTYwNzkyNDUwLDE2MDc5MjQ0OSwxNjA3OTI0MzUsMTYwNzkyNDI4LDE2MDc5MjQyNywxNjA3OTI0MjYsMTYwNzkyNDI1LDE2MDc5MjQyNCwxNjA3OTI0MjMsMTYwNzkyNDIyLDE2MDc5MjQxNCwxNjA3OTI0MTEsMTYwNzkyNDA4LDE2MDc5MjQwMywxNjA3OTI0MDIsMTYwNzkyNDAxLDE2MDc5MjM5MiwxNjA3OTIzODgsMTYwNzkyMzg3LDE2MDc5MjM4NiwxNjA3OTIzODUsMTYwNzkyMzg0LDE2MDc5MjM2NSwxNjA3OTIzNjIsMTYwNzkyMzYxLDE2MDc5MjM2MCwxNjA3OTIzNTksMTYwNzkyMzU4LDE2MDc5MjM1NywxNjA3OTIzNTYsMTYwNzkyMzU1LDE2MDc5MjM1NCwxNjA3OTIzNTMsMTYwNzkyMzUyLDE2MDc5MjM1MSwxNjA3OTIzNTAsMTYwNzkwNjA4LDE2MDc5MDYwNywxNjA3OTA2MDQsMTYwNzkwNjAzLDE2MDc5MDYwMCwxNjA3OTA1OTYsMTYwNzkwNTk1LDE2MDc5MDU5NCwxNjA3Nzg3MjQsMTYwNzc4NzIzLDE2MDc3ODcyMiwxNjA3Nzg3MjEsMTYwNzc4NzIwLDE2MDc3ODcxOSwxNjA3Nzg3MTgsMTYwNzc4NzE3LDE2MDc3ODcxNiwxNjA3Nzg3MTUsMTYwNzc4NzE0LDE2MDc3ODcxMywxNjA3Nzg3MTIsMTYwNzY4MjMxLDE2MDc2NDE3NywxNjA3NjQxNTgsMTYwNzYyNTI0LDE2MDc2MjUyMywxNjA3NjI1MjIsMTYwNzYyNTIxLDE2MDc2MjUyMCwxNjA3NjI1MTksMTYwNzYyNTE4LDE2MDc2MjUxNywxNjA3NjI1MTYsMTYwNzYyNTE1LDE2MDc2MjQ5MiwxNjA3NjI0OTEsMTYwNzYyNDkwLDE2MDc2MjQ4OSwxNjA3NjI0ODgsMTYwNzYyNDg3LDE2MDc2MjQ4NiwxNjA3NTY4MDksMTYwNzU2Nzc5LDE2MDc1Njc0OSwxNjA3NTY3NDgsMTYwNzU2NzM4LDE2MDc1NjczNywxNjA3NTY3MjAsMTYwNzU2NzE5LDE2MDc1NjcxOCwxNjA3NTY3MTcsMTYwNzU2Njk1LDE2MDczNzIzNSwxNjA3MzcyMzQsMTYwNzM3MjMzLDE2MDczNzIzMiwxNjA3MzcyMzEsMTYwNzM3MjMwLDE2MDczNzIyOSwxNjA3MzcyMjgsMTYwNzM3MjI3LDE2MDczNzIyNiwxNjA3MzcyMjUsMTYwNzM3MjI0LDE2MDczNzE3MywxNjA3MzcxNzJdfX0seyJ0ZXJtIjp7ImlzX2RlbGV0ZWQiOjB9fV19fQ=="}','$.key')
{code}

result will be truncated in spark-sql.

{code:java}
val spark = SparkSession.builder
  .appName("test").master("local[1]")
  .getOrCreate
import spark.implicits._ // << add this

val sc = spark.sparkContext
val rdd: RDD[Row] = sc.parallelize(Seq(
  Row("""{"key":"eyJib29sIjp7ImZpbHRlciI6W3sidGVybSI6eyJ3YXJlaG91c2VfaWQiOjMwfX0seyJ0ZXJtcyI6eyJpZCI6WzE2MTgzMjk3NywxNjE4MjM1OTgsMTYxODIzNTk3LDE2MTgyMzU5NiwxNjE4MjM1OTUsMTYxODAwMjI0LDE2MTgwMDIyMSwxNjE4MDAyMjAsMTYxODAwMjE5LDE2MTgwMDIxOCwxNjE4MDAyMTMsMTYxODAwMjEyLDE2MTgwMDIxMSwxNjE4MDAyMTAsMTYxODAwMjA2LDE2MTgwMDIwMiwxNjE4MDAyMDEsMTYxODAwMTk4LDE2MTgwMDE5NywxNjE4MDAxOTYsMTYxODAwMTk1LDE2MTgwMDE5NCwxNjE4MDAxOTMsMTYxODAwMTkyLDE2MTgwMDE5MSwxNjE4MDAxOTAsMTYxODAwMTg5LDE2MTc5NDAyOSwxNjEzNTg2ODMsMTYxMzU4NjExLDE2MTM1ODYxMCwxNjEzNTYzMTIsMTYxMzU0NzU5LDE2MTM1NDY5NywxNjEzNTQ2OTYsMTYxMzQ0NTkyLDE2MTM0NDUwNywxNjEzMzc2MzQsMTYxMzM3MjA2LDE2MTMwMDM4OCwxNjEzMDAzODcsMTYxMzAwMzg2LDE2MTI4ODAxMCwxNjEyODgwMDksMTYxMjc5NTI0LDE2MTI3OTUyMiwxNjEyNzk1MTUsMTYxMjc5NTE0LDE2MTI3OTUxMywxNjEyNzg5NzcsMTYxMjc4OTc2LDE2MTI3ODk0NCwxNjEyNzg5NDMsMTYwODk2NjM3LDE2MDg5NjYzNiwxNjA4NzQ0ODcsMTYwODc0NDg2LDE2MDg3NDQ2NywxNjA4NzQ0NjYsMTYwODc0NDY1LDE2MDg3NDQ2NCwxNjA4NzQ0NjMsMTYwODc0NDU2LDE2MDg3NDQ1NSwxNjA4NzQxNTksMTYwODc0MTU2LDE2MDg3NDE1NSwxNjA4NzQxNTIsMTYwODc0MTUxLDE2MDg3NDE1MCwxNjA4NzQxNDksMTYwODc0MTQ4LDE2MDg3NDE0NSwxNjA4NzQxNDQsMTYwODc0MTQzLDE2MDg3NDE0MiwxNjA4NzQxNDEsMTYwODc0MTQwLDE2MDg3NDEzNywxNjA4NzQxMzYsMTYwODc0MTMzLDE2MDg3NDEzMiwxNjA4MjI0OTksMTYwODExMDE5LDE2MDgxMDkxNSwxNjA4MTA5MTQsMTYwODEwNDU4LDE2MDgxMDQ1NywxNjA4MTA0NTYsMTYwODEwNDU1LDE2MDgxMDQ1NCwxNjA4MTAzNTMsMTYwNzk5NTU1LDE2MDc5MjQ1OSwxNjA3OTI0NTYsMTYwNzkyNDU1LDE2MDc5MjQ1NCwxNjA3OTI0NTEsMTYwNzkyNDUwLDE2MDc5MjQ0OSwxNjA3OTI0MzUsMTYwNzkyNDI4LDE2MDc5MjQyNywxNjA3OTI0MjYsMTYwNzkyNDI1LDE2MDc5MjQyNCwxNjA3OTI0MjMsMTYwNzkyNDIyLDE2MDc5MjQxNCwxNjA3OTI0MTEsMTYwNzkyNDA4LDE2MDc5MjQwMywxNjA3OTI0MDIsMTYwNzkyNDAxLDE2MDc5MjM5MiwxNjA3OTIzODgsMTYwNzkyMzg3LDE2MDc5MjM4NiwxNjA3OTIzODUsMTYwNzkyMzg0LDE2MDc5MjM2NSwxNjA3OTIzNjIsMTYwNzkyMzYxLDE2MDc5MjM2MCwxNjA3OTIzNTksMTYwNzkyMzU4LDE2MDc5MjM1NywxNjA3OTIzNTYsMTYwNzkyMzU1LDE2MDc5MjM1NCwxNjA3OTIzNTMsMTYwNzkyMzUyLDE2MDc5MjM1MSwxNjA3OTIzNTAsMTYwNzkwNjA4LDE2MDc5MDYwNywxNjA3OTA2MDQsMTYwNzkwNjAzLDE2MDc5MDYwMCwxNjA3OTA1OTYsMTYwNzkwNTk1LDE2MDc5MDU5NCwxNjA3Nzg3MjQsMTYwNzc4NzIzLDE2MDc3ODcyMiwxNjA3Nzg3MjEsMTYwNzc4NzIwLDE2MDc3ODcxOSwxNjA3Nzg3MTgsMTYwNzc4NzE3LDE2MDc3ODcxNiwxNjA3Nzg3MTUsMTYwNzc4NzE0LDE2MDc3ODcxMywxNjA3Nzg3MTIsMTYwNzY4MjMxLDE2MDc2NDE3NywxNjA3NjQxNTgsMTYwNzYyNTI0LDE2MDc2MjUyMywxNjA3NjI1MjIsMTYwNzYyNTIxLDE2MDc2MjUyMCwxNjA3NjI1MTksMTYwNzYyNTE4LDE2MDc2MjUxNywxNjA3NjI1MTYsMTYwNzYyNTE1LDE2MDc2MjQ5MiwxNjA3NjI0OTEsMTYwNzYyNDkwLDE2MDc2MjQ4OSwxNjA3NjI0ODgsMTYwNzYyNDg3LDE2MDc2MjQ4NiwxNjA3NTY4MDksMTYwNzU2Nzc5LDE2MDc1Njc0OSwxNjA3NTY3NDgsMTYwNzU2NzM4LDE2MDc1NjczNywxNjA3NTY3MjAsMTYwNzU2NzE5LDE2MDc1NjcxOCwxNjA3NTY3MTcsMTYwNzU2Njk1LDE2MDczNzIzNSwxNjA3MzcyMzQsMTYwNzM3MjMzLDE2MDczNzIzMiwxNjA3MzcyMzEsMTYwNzM3MjMwLDE2MDczNzIyOSwxNjA3MzcyMjgsMTYwNzM3MjI3LDE2MDczNzIyNiwxNjA3MzcyMjUsMTYwNzM3MjI0LDE2MDczNzE3MywxNjA3MzcxNzJdfX0seyJ0ZXJtIjp7ImlzX2RlbGV0ZWQiOjB9fV19fQ=="}""")
))
val schema = StructType(
  List(
    StructField("x", StringType, true)
  ))
val df = spark.createDataFrame(rdd, schema)

df.withColumn("x2", get_json_object( $"x"
, "$.key")).select("x2").show(false) 
{code}

can result complete value, why does spark-sql behave differently with the program?

  was:
{code:java}
select get_json_object('{"key":"eyJib29sIjp7ImZpbHRlciI6W3sidGVybSI6eyJ3YXJlaG91c2VfaWQiOjMwfX0seyJ0ZXJtcyI6eyJpZCI6WzE2MTgzMjk3NywxNjE4MjM1OTgsMTYxODIzNTk3LDE2MTgyMzU5NiwxNjE4MjM1OTUsMTYxODAwMjI0LDE2MTgwMDIyMSwxNjE4MDAyMjAsMTYxODAwMjE5LDE2MTgwMDIxOCwxNjE4MDAyMTMsMTYxODAwMjEyLDE2MTgwMDIxMSwxNjE4MDAyMTAsMTYxODAwMjA2LDE2MTgwMDIwMiwxNjE4MDAyMDEsMTYxODAwMTk4LDE2MTgwMDE5NywxNjE4MDAxOTYsMTYxODAwMTk1LDE2MTgwMDE5NCwxNjE4MDAxOTMsMTYxODAwMTkyLDE2MTgwMDE5MSwxNjE4MDAxOTAsMTYxODAwMTg5LDE2MTc5NDAyOSwxNjEzNTg2ODMsMTYxMzU4NjExLDE2MTM1ODYxMCwxNjEzNTYzMTIsMTYxMzU0NzU5LDE2MTM1NDY5NywxNjEzNTQ2OTYsMTYxMzQ0NTkyLDE2MTM0NDUwNywxNjEzMzc2MzQsMTYxMzM3MjA2LDE2MTMwMDM4OCwxNjEzMDAzODcsMTYxMzAwMzg2LDE2MTI4ODAxMCwxNjEyODgwMDksMTYxMjc5NTI0LDE2MTI3OTUyMiwxNjEyNzk1MTUsMTYxMjc5NTE0LDE2MTI3OTUxMywxNjEyNzg5NzcsMTYxMjc4OTc2LDE2MTI3ODk0NCwxNjEyNzg5NDMsMTYwODk2NjM3LDE2MDg5NjYzNiwxNjA4NzQ0ODcsMTYwODc0NDg2LDE2MDg3NDQ2NywxNjA4NzQ0NjYsMTYwODc0NDY1LDE2MDg3NDQ2NCwxNjA4NzQ0NjMsMTYwODc0NDU2LDE2MDg3NDQ1NSwxNjA4NzQxNTksMTYwODc0MTU2LDE2MDg3NDE1NSwxNjA4NzQxNTIsMTYwODc0MTUxLDE2MDg3NDE1MCwxNjA4NzQxNDksMTYwODc0MTQ4LDE2MDg3NDE0NSwxNjA4NzQxNDQsMTYwODc0MTQzLDE2MDg3NDE0MiwxNjA4NzQxNDEsMTYwODc0MTQwLDE2MDg3NDEzNywxNjA4NzQxMzYsMTYwODc0MTMzLDE2MDg3NDEzMiwxNjA4MjI0OTksMTYwODExMDE5LDE2MDgxMDkxNSwxNjA4MTA5MTQsMTYwODEwNDU4LDE2MDgxMDQ1NywxNjA4MTA0NTYsMTYwODEwNDU1LDE2MDgxMDQ1NCwxNjA4MTAzNTMsMTYwNzk5NTU1LDE2MDc5MjQ1OSwxNjA3OTI0NTYsMTYwNzkyNDU1LDE2MDc5MjQ1NCwxNjA3OTI0NTEsMTYwNzkyNDUwLDE2MDc5MjQ0OSwxNjA3OTI0MzUsMTYwNzkyNDI4LDE2MDc5MjQyNywxNjA3OTI0MjYsMTYwNzkyNDI1LDE2MDc5MjQyNCwxNjA3OTI0MjMsMTYwNzkyNDIyLDE2MDc5MjQxNCwxNjA3OTI0MTEsMTYwNzkyNDA4LDE2MDc5MjQwMywxNjA3OTI0MDIsMTYwNzkyNDAxLDE2MDc5MjM5MiwxNjA3OTIzODgsMTYwNzkyMzg3LDE2MDc5MjM4NiwxNjA3OTIzODUsMTYwNzkyMzg0LDE2MDc5MjM2NSwxNjA3OTIzNjIsMTYwNzkyMzYxLDE2MDc5MjM2MCwxNjA3OTIzNTksMTYwNzkyMzU4LDE2MDc5MjM1NywxNjA3OTIzNTYsMTYwNzkyMzU1LDE2MDc5MjM1NCwxNjA3OTIzNTMsMTYwNzkyMzUyLDE2MDc5MjM1MSwxNjA3OTIzNTAsMTYwNzkwNjA4LDE2MDc5MDYwNywxNjA3OTA2MDQsMTYwNzkwNjAzLDE2MDc5MDYwMCwxNjA3OTA1OTYsMTYwNzkwNTk1LDE2MDc5MDU5NCwxNjA3Nzg3MjQsMTYwNzc4NzIzLDE2MDc3ODcyMiwxNjA3Nzg3MjEsMTYwNzc4NzIwLDE2MDc3ODcxOSwxNjA3Nzg3MTgsMTYwNzc4NzE3LDE2MDc3ODcxNiwxNjA3Nzg3MTUsMTYwNzc4NzE0LDE2MDc3ODcxMywxNjA3Nzg3MTIsMTYwNzY4MjMxLDE2MDc2NDE3NywxNjA3NjQxNTgsMTYwNzYyNTI0LDE2MDc2MjUyMywxNjA3NjI1MjIsMTYwNzYyNTIxLDE2MDc2MjUyMCwxNjA3NjI1MTksMTYwNzYyNTE4LDE2MDc2MjUxNywxNjA3NjI1MTYsMTYwNzYyNTE1LDE2MDc2MjQ5MiwxNjA3NjI0OTEsMTYwNzYyNDkwLDE2MDc2MjQ4OSwxNjA3NjI0ODgsMTYwNzYyNDg3LDE2MDc2MjQ4NiwxNjA3NTY4MDksMTYwNzU2Nzc5LDE2MDc1Njc0OSwxNjA3NTY3NDgsMTYwNzU2NzM4LDE2MDc1NjczNywxNjA3NTY3MjAsMTYwNzU2NzE5LDE2MDc1NjcxOCwxNjA3NTY3MTcsMTYwNzU2Njk1LDE2MDczNzIzNSwxNjA3MzcyMzQsMTYwNzM3MjMzLDE2MDczNzIzMiwxNjA3MzcyMzEsMTYwNzM3MjMwLDE2MDczNzIyOSwxNjA3MzcyMjgsMTYwNzM3MjI3LDE2MDczNzIyNiwxNjA3MzcyMjUsMTYwNzM3MjI0LDE2MDczNzE3MywxNjA3MzcxNzJdfX0seyJ0ZXJtIjp7ImlzX2RlbGV0ZWQiOjB9fV19fQ=="}','$.key') result will be truncated in spark-sql.
{code}
{code:java}
val spark = SparkSession.builder
  .appName("test").master("local[1]")
  .getOrCreate
import spark.implicits._ // << add this

val sc = spark.sparkContext
val rdd: RDD[Row] = sc.parallelize(Seq(
  Row("""{"key":"eyJib29sIjp7ImZpbHRlciI6W3sidGVybSI6eyJ3YXJlaG91c2VfaWQiOjMwfX0seyJ0ZXJtcyI6eyJpZCI6WzE2MTgzMjk3NywxNjE4MjM1OTgsMTYxODIzNTk3LDE2MTgyMzU5NiwxNjE4MjM1OTUsMTYxODAwMjI0LDE2MTgwMDIyMSwxNjE4MDAyMjAsMTYxODAwMjE5LDE2MTgwMDIxOCwxNjE4MDAyMTMsMTYxODAwMjEyLDE2MTgwMDIxMSwxNjE4MDAyMTAsMTYxODAwMjA2LDE2MTgwMDIwMiwxNjE4MDAyMDEsMTYxODAwMTk4LDE2MTgwMDE5NywxNjE4MDAxOTYsMTYxODAwMTk1LDE2MTgwMDE5NCwxNjE4MDAxOTMsMTYxODAwMTkyLDE2MTgwMDE5MSwxNjE4MDAxOTAsMTYxODAwMTg5LDE2MTc5NDAyOSwxNjEzNTg2ODMsMTYxMzU4NjExLDE2MTM1ODYxMCwxNjEzNTYzMTIsMTYxMzU0NzU5LDE2MTM1NDY5NywxNjEzNTQ2OTYsMTYxMzQ0NTkyLDE2MTM0NDUwNywxNjEzMzc2MzQsMTYxMzM3MjA2LDE2MTMwMDM4OCwxNjEzMDAzODcsMTYxMzAwMzg2LDE2MTI4ODAxMCwxNjEyODgwMDksMTYxMjc5NTI0LDE2MTI3OTUyMiwxNjEyNzk1MTUsMTYxMjc5NTE0LDE2MTI3OTUxMywxNjEyNzg5NzcsMTYxMjc4OTc2LDE2MTI3ODk0NCwxNjEyNzg5NDMsMTYwODk2NjM3LDE2MDg5NjYzNiwxNjA4NzQ0ODcsMTYwODc0NDg2LDE2MDg3NDQ2NywxNjA4NzQ0NjYsMTYwODc0NDY1LDE2MDg3NDQ2NCwxNjA4NzQ0NjMsMTYwODc0NDU2LDE2MDg3NDQ1NSwxNjA4NzQxNTksMTYwODc0MTU2LDE2MDg3NDE1NSwxNjA4NzQxNTIsMTYwODc0MTUxLDE2MDg3NDE1MCwxNjA4NzQxNDksMTYwODc0MTQ4LDE2MDg3NDE0NSwxNjA4NzQxNDQsMTYwODc0MTQzLDE2MDg3NDE0MiwxNjA4NzQxNDEsMTYwODc0MTQwLDE2MDg3NDEzNywxNjA4NzQxMzYsMTYwODc0MTMzLDE2MDg3NDEzMiwxNjA4MjI0OTksMTYwODExMDE5LDE2MDgxMDkxNSwxNjA4MTA5MTQsMTYwODEwNDU4LDE2MDgxMDQ1NywxNjA4MTA0NTYsMTYwODEwNDU1LDE2MDgxMDQ1NCwxNjA4MTAzNTMsMTYwNzk5NTU1LDE2MDc5MjQ1OSwxNjA3OTI0NTYsMTYwNzkyNDU1LDE2MDc5MjQ1NCwxNjA3OTI0NTEsMTYwNzkyNDUwLDE2MDc5MjQ0OSwxNjA3OTI0MzUsMTYwNzkyNDI4LDE2MDc5MjQyNywxNjA3OTI0MjYsMTYwNzkyNDI1LDE2MDc5MjQyNCwxNjA3OTI0MjMsMTYwNzkyNDIyLDE2MDc5MjQxNCwxNjA3OTI0MTEsMTYwNzkyNDA4LDE2MDc5MjQwMywxNjA3OTI0MDIsMTYwNzkyNDAxLDE2MDc5MjM5MiwxNjA3OTIzODgsMTYwNzkyMzg3LDE2MDc5MjM4NiwxNjA3OTIzODUsMTYwNzkyMzg0LDE2MDc5MjM2NSwxNjA3OTIzNjIsMTYwNzkyMzYxLDE2MDc5MjM2MCwxNjA3OTIzNTksMTYwNzkyMzU4LDE2MDc5MjM1NywxNjA3OTIzNTYsMTYwNzkyMzU1LDE2MDc5MjM1NCwxNjA3OTIzNTMsMTYwNzkyMzUyLDE2MDc5MjM1MSwxNjA3OTIzNTAsMTYwNzkwNjA4LDE2MDc5MDYwNywxNjA3OTA2MDQsMTYwNzkwNjAzLDE2MDc5MDYwMCwxNjA3OTA1OTYsMTYwNzkwNTk1LDE2MDc5MDU5NCwxNjA3Nzg3MjQsMTYwNzc4NzIzLDE2MDc3ODcyMiwxNjA3Nzg3MjEsMTYwNzc4NzIwLDE2MDc3ODcxOSwxNjA3Nzg3MTgsMTYwNzc4NzE3LDE2MDc3ODcxNiwxNjA3Nzg3MTUsMTYwNzc4NzE0LDE2MDc3ODcxMywxNjA3Nzg3MTIsMTYwNzY4MjMxLDE2MDc2NDE3NywxNjA3NjQxNTgsMTYwNzYyNTI0LDE2MDc2MjUyMywxNjA3NjI1MjIsMTYwNzYyNTIxLDE2MDc2MjUyMCwxNjA3NjI1MTksMTYwNzYyNTE4LDE2MDc2MjUxNywxNjA3NjI1MTYsMTYwNzYyNTE1LDE2MDc2MjQ5MiwxNjA3NjI0OTEsMTYwNzYyNDkwLDE2MDc2MjQ4OSwxNjA3NjI0ODgsMTYwNzYyNDg3LDE2MDc2MjQ4NiwxNjA3NTY4MDksMTYwNzU2Nzc5LDE2MDc1Njc0OSwxNjA3NTY3NDgsMTYwNzU2NzM4LDE2MDc1NjczNywxNjA3NTY3MjAsMTYwNzU2NzE5LDE2MDc1NjcxOCwxNjA3NTY3MTcsMTYwNzU2Njk1LDE2MDczNzIzNSwxNjA3MzcyMzQsMTYwNzM3MjMzLDE2MDczNzIzMiwxNjA3MzcyMzEsMTYwNzM3MjMwLDE2MDczNzIyOSwxNjA3MzcyMjgsMTYwNzM3MjI3LDE2MDczNzIyNiwxNjA3MzcyMjUsMTYwNzM3MjI0LDE2MDczNzE3MywxNjA3MzcxNzJdfX0seyJ0ZXJtIjp7ImlzX2RlbGV0ZWQiOjB9fV19fQ=="}""")
))
val schema = StructType(
  List(
    StructField("x", StringType, true)
  ))
val df = spark.createDataFrame(rdd, schema)

df.withColumn("x2", get_json_object( $"x"
, "$.key")).select("x2").show(false) 

can result complete value, why does spark-sql behave differently with the program?
{code}


> get_json_object in sql will truncate long value gotten from jsonpath 
> ---------------------------------------------------------------------
>
>                 Key: SPARK-27861
>                 URL: https://issues.apache.org/jira/browse/SPARK-27861
>             Project: Spark
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 2.2.0
>            Reporter: jing.yan
>            Priority: Major
>
> {code:java}
> select get_json_object('{"key":"eyJib29sIjp7ImZpbHRlciI6W3sidGVybSI6eyJ3YXJlaG91c2VfaWQiOjMwfX0seyJ0ZXJtcyI6eyJpZCI6WzE2MTgzMjk3NywxNjE4MjM1OTgsMTYxODIzNTk3LDE2MTgyMzU5NiwxNjE4MjM1OTUsMTYxODAwMjI0LDE2MTgwMDIyMSwxNjE4MDAyMjAsMTYxODAwMjE5LDE2MTgwMDIxOCwxNjE4MDAyMTMsMTYxODAwMjEyLDE2MTgwMDIxMSwxNjE4MDAyMTAsMTYxODAwMjA2LDE2MTgwMDIwMiwxNjE4MDAyMDEsMTYxODAwMTk4LDE2MTgwMDE5NywxNjE4MDAxOTYsMTYxODAwMTk1LDE2MTgwMDE5NCwxNjE4MDAxOTMsMTYxODAwMTkyLDE2MTgwMDE5MSwxNjE4MDAxOTAsMTYxODAwMTg5LDE2MTc5NDAyOSwxNjEzNTg2ODMsMTYxMzU4NjExLDE2MTM1ODYxMCwxNjEzNTYzMTIsMTYxMzU0NzU5LDE2MTM1NDY5NywxNjEzNTQ2OTYsMTYxMzQ0NTkyLDE2MTM0NDUwNywxNjEzMzc2MzQsMTYxMzM3MjA2LDE2MTMwMDM4OCwxNjEzMDAzODcsMTYxMzAwMzg2LDE2MTI4ODAxMCwxNjEyODgwMDksMTYxMjc5NTI0LDE2MTI3OTUyMiwxNjEyNzk1MTUsMTYxMjc5NTE0LDE2MTI3OTUxMywxNjEyNzg5NzcsMTYxMjc4OTc2LDE2MTI3ODk0NCwxNjEyNzg5NDMsMTYwODk2NjM3LDE2MDg5NjYzNiwxNjA4NzQ0ODcsMTYwODc0NDg2LDE2MDg3NDQ2NywxNjA4NzQ0NjYsMTYwODc0NDY1LDE2MDg3NDQ2NCwxNjA4NzQ0NjMsMTYwODc0NDU2LDE2MDg3NDQ1NSwxNjA4NzQxNTksMTYwODc0MTU2LDE2MDg3NDE1NSwxNjA4NzQxNTIsMTYwODc0MTUxLDE2MDg3NDE1MCwxNjA4NzQxNDksMTYwODc0MTQ4LDE2MDg3NDE0NSwxNjA4NzQxNDQsMTYwODc0MTQzLDE2MDg3NDE0MiwxNjA4NzQxNDEsMTYwODc0MTQwLDE2MDg3NDEzNywxNjA4NzQxMzYsMTYwODc0MTMzLDE2MDg3NDEzMiwxNjA4MjI0OTksMTYwODExMDE5LDE2MDgxMDkxNSwxNjA4MTA5MTQsMTYwODEwNDU4LDE2MDgxMDQ1NywxNjA4MTA0NTYsMTYwODEwNDU1LDE2MDgxMDQ1NCwxNjA4MTAzNTMsMTYwNzk5NTU1LDE2MDc5MjQ1OSwxNjA3OTI0NTYsMTYwNzkyNDU1LDE2MDc5MjQ1NCwxNjA3OTI0NTEsMTYwNzkyNDUwLDE2MDc5MjQ0OSwxNjA3OTI0MzUsMTYwNzkyNDI4LDE2MDc5MjQyNywxNjA3OTI0MjYsMTYwNzkyNDI1LDE2MDc5MjQyNCwxNjA3OTI0MjMsMTYwNzkyNDIyLDE2MDc5MjQxNCwxNjA3OTI0MTEsMTYwNzkyNDA4LDE2MDc5MjQwMywxNjA3OTI0MDIsMTYwNzkyNDAxLDE2MDc5MjM5MiwxNjA3OTIzODgsMTYwNzkyMzg3LDE2MDc5MjM4NiwxNjA3OTIzODUsMTYwNzkyMzg0LDE2MDc5MjM2NSwxNjA3OTIzNjIsMTYwNzkyMzYxLDE2MDc5MjM2MCwxNjA3OTIzNTksMTYwNzkyMzU4LDE2MDc5MjM1NywxNjA3OTIzNTYsMTYwNzkyMzU1LDE2MDc5MjM1NCwxNjA3OTIzNTMsMTYwNzkyMzUyLDE2MDc5MjM1MSwxNjA3OTIzNTAsMTYwNzkwNjA4LDE2MDc5MDYwNywxNjA3OTA2MDQsMTYwNzkwNjAzLDE2MDc5MDYwMCwxNjA3OTA1OTYsMTYwNzkwNTk1LDE2MDc5MDU5NCwxNjA3Nzg3MjQsMTYwNzc4NzIzLDE2MDc3ODcyMiwxNjA3Nzg3MjEsMTYwNzc4NzIwLDE2MDc3ODcxOSwxNjA3Nzg3MTgsMTYwNzc4NzE3LDE2MDc3ODcxNiwxNjA3Nzg3MTUsMTYwNzc4NzE0LDE2MDc3ODcxMywxNjA3Nzg3MTIsMTYwNzY4MjMxLDE2MDc2NDE3NywxNjA3NjQxNTgsMTYwNzYyNTI0LDE2MDc2MjUyMywxNjA3NjI1MjIsMTYwNzYyNTIxLDE2MDc2MjUyMCwxNjA3NjI1MTksMTYwNzYyNTE4LDE2MDc2MjUxNywxNjA3NjI1MTYsMTYwNzYyNTE1LDE2MDc2MjQ5MiwxNjA3NjI0OTEsMTYwNzYyNDkwLDE2MDc2MjQ4OSwxNjA3NjI0ODgsMTYwNzYyNDg3LDE2MDc2MjQ4NiwxNjA3NTY4MDksMTYwNzU2Nzc5LDE2MDc1Njc0OSwxNjA3NTY3NDgsMTYwNzU2NzM4LDE2MDc1NjczNywxNjA3NTY3MjAsMTYwNzU2NzE5LDE2MDc1NjcxOCwxNjA3NTY3MTcsMTYwNzU2Njk1LDE2MDczNzIzNSwxNjA3MzcyMzQsMTYwNzM3MjMzLDE2MDczNzIzMiwxNjA3MzcyMzEsMTYwNzM3MjMwLDE2MDczNzIyOSwxNjA3MzcyMjgsMTYwNzM3MjI3LDE2MDczNzIyNiwxNjA3MzcyMjUsMTYwNzM3MjI0LDE2MDczNzE3MywxNjA3MzcxNzJdfX0seyJ0ZXJtIjp7ImlzX2RlbGV0ZWQiOjB9fV19fQ=="}','$.key')
> {code}
> result will be truncated in spark-sql.
> {code:java}
> val spark = SparkSession.builder
>   .appName("test").master("local[1]")
>   .getOrCreate
> import spark.implicits._ // << add this
> val sc = spark.sparkContext
> val rdd: RDD[Row] = sc.parallelize(Seq(
>   Row("""{"key":"eyJib29sIjp7ImZpbHRlciI6W3sidGVybSI6eyJ3YXJlaG91c2VfaWQiOjMwfX0seyJ0ZXJtcyI6eyJpZCI6WzE2MTgzMjk3NywxNjE4MjM1OTgsMTYxODIzNTk3LDE2MTgyMzU5NiwxNjE4MjM1OTUsMTYxODAwMjI0LDE2MTgwMDIyMSwxNjE4MDAyMjAsMTYxODAwMjE5LDE2MTgwMDIxOCwxNjE4MDAyMTMsMTYxODAwMjEyLDE2MTgwMDIxMSwxNjE4MDAyMTAsMTYxODAwMjA2LDE2MTgwMDIwMiwxNjE4MDAyMDEsMTYxODAwMTk4LDE2MTgwMDE5NywxNjE4MDAxOTYsMTYxODAwMTk1LDE2MTgwMDE5NCwxNjE4MDAxOTMsMTYxODAwMTkyLDE2MTgwMDE5MSwxNjE4MDAxOTAsMTYxODAwMTg5LDE2MTc5NDAyOSwxNjEzNTg2ODMsMTYxMzU4NjExLDE2MTM1ODYxMCwxNjEzNTYzMTIsMTYxMzU0NzU5LDE2MTM1NDY5NywxNjEzNTQ2OTYsMTYxMzQ0NTkyLDE2MTM0NDUwNywxNjEzMzc2MzQsMTYxMzM3MjA2LDE2MTMwMDM4OCwxNjEzMDAzODcsMTYxMzAwMzg2LDE2MTI4ODAxMCwxNjEyODgwMDksMTYxMjc5NTI0LDE2MTI3OTUyMiwxNjEyNzk1MTUsMTYxMjc5NTE0LDE2MTI3OTUxMywxNjEyNzg5NzcsMTYxMjc4OTc2LDE2MTI3ODk0NCwxNjEyNzg5NDMsMTYwODk2NjM3LDE2MDg5NjYzNiwxNjA4NzQ0ODcsMTYwODc0NDg2LDE2MDg3NDQ2NywxNjA4NzQ0NjYsMTYwODc0NDY1LDE2MDg3NDQ2NCwxNjA4NzQ0NjMsMTYwODc0NDU2LDE2MDg3NDQ1NSwxNjA4NzQxNTksMTYwODc0MTU2LDE2MDg3NDE1NSwxNjA4NzQxNTIsMTYwODc0MTUxLDE2MDg3NDE1MCwxNjA4NzQxNDksMTYwODc0MTQ4LDE2MDg3NDE0NSwxNjA4NzQxNDQsMTYwODc0MTQzLDE2MDg3NDE0MiwxNjA4NzQxNDEsMTYwODc0MTQwLDE2MDg3NDEzNywxNjA4NzQxMzYsMTYwODc0MTMzLDE2MDg3NDEzMiwxNjA4MjI0OTksMTYwODExMDE5LDE2MDgxMDkxNSwxNjA4MTA5MTQsMTYwODEwNDU4LDE2MDgxMDQ1NywxNjA4MTA0NTYsMTYwODEwNDU1LDE2MDgxMDQ1NCwxNjA4MTAzNTMsMTYwNzk5NTU1LDE2MDc5MjQ1OSwxNjA3OTI0NTYsMTYwNzkyNDU1LDE2MDc5MjQ1NCwxNjA3OTI0NTEsMTYwNzkyNDUwLDE2MDc5MjQ0OSwxNjA3OTI0MzUsMTYwNzkyNDI4LDE2MDc5MjQyNywxNjA3OTI0MjYsMTYwNzkyNDI1LDE2MDc5MjQyNCwxNjA3OTI0MjMsMTYwNzkyNDIyLDE2MDc5MjQxNCwxNjA3OTI0MTEsMTYwNzkyNDA4LDE2MDc5MjQwMywxNjA3OTI0MDIsMTYwNzkyNDAxLDE2MDc5MjM5MiwxNjA3OTIzODgsMTYwNzkyMzg3LDE2MDc5MjM4NiwxNjA3OTIzODUsMTYwNzkyMzg0LDE2MDc5MjM2NSwxNjA3OTIzNjIsMTYwNzkyMzYxLDE2MDc5MjM2MCwxNjA3OTIzNTksMTYwNzkyMzU4LDE2MDc5MjM1NywxNjA3OTIzNTYsMTYwNzkyMzU1LDE2MDc5MjM1NCwxNjA3OTIzNTMsMTYwNzkyMzUyLDE2MDc5MjM1MSwxNjA3OTIzNTAsMTYwNzkwNjA4LDE2MDc5MDYwNywxNjA3OTA2MDQsMTYwNzkwNjAzLDE2MDc5MDYwMCwxNjA3OTA1OTYsMTYwNzkwNTk1LDE2MDc5MDU5NCwxNjA3Nzg3MjQsMTYwNzc4NzIzLDE2MDc3ODcyMiwxNjA3Nzg3MjEsMTYwNzc4NzIwLDE2MDc3ODcxOSwxNjA3Nzg3MTgsMTYwNzc4NzE3LDE2MDc3ODcxNiwxNjA3Nzg3MTUsMTYwNzc4NzE0LDE2MDc3ODcxMywxNjA3Nzg3MTIsMTYwNzY4MjMxLDE2MDc2NDE3NywxNjA3NjQxNTgsMTYwNzYyNTI0LDE2MDc2MjUyMywxNjA3NjI1MjIsMTYwNzYyNTIxLDE2MDc2MjUyMCwxNjA3NjI1MTksMTYwNzYyNTE4LDE2MDc2MjUxNywxNjA3NjI1MTYsMTYwNzYyNTE1LDE2MDc2MjQ5MiwxNjA3NjI0OTEsMTYwNzYyNDkwLDE2MDc2MjQ4OSwxNjA3NjI0ODgsMTYwNzYyNDg3LDE2MDc2MjQ4NiwxNjA3NTY4MDksMTYwNzU2Nzc5LDE2MDc1Njc0OSwxNjA3NTY3NDgsMTYwNzU2NzM4LDE2MDc1NjczNywxNjA3NTY3MjAsMTYwNzU2NzE5LDE2MDc1NjcxOCwxNjA3NTY3MTcsMTYwNzU2Njk1LDE2MDczNzIzNSwxNjA3MzcyMzQsMTYwNzM3MjMzLDE2MDczNzIzMiwxNjA3MzcyMzEsMTYwNzM3MjMwLDE2MDczNzIyOSwxNjA3MzcyMjgsMTYwNzM3MjI3LDE2MDczNzIyNiwxNjA3MzcyMjUsMTYwNzM3MjI0LDE2MDczNzE3MywxNjA3MzcxNzJdfX0seyJ0ZXJtIjp7ImlzX2RlbGV0ZWQiOjB9fV19fQ=="}""")
> ))
> val schema = StructType(
>   List(
>     StructField("x", StringType, true)
>   ))
> val df = spark.createDataFrame(rdd, schema)
> df.withColumn("x2", get_json_object( $"x"
> , "$.key")).select("x2").show(false) 
> {code}
> can result complete value, why does spark-sql behave differently with the program?



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org