You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Davies Liu (JIRA)" <ji...@apache.org> on 2015/09/28 21:05:04 UTC
[jira] [Created] (SPARK-10859) Predicates pushed to
InmemoryColumnarTableScan are not evaluated correctly
Davies Liu created SPARK-10859:
----------------------------------
Summary: Predicates pushed to InmemoryColumnarTableScan are not evaluated correctly
Key: SPARK-10859
URL: https://issues.apache.org/jira/browse/SPARK-10859
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 1.5.0, 1.5.1
Reporter: Davies Liu
Assignee: Davies Liu
Priority: Blocker
{code}
var data01 = sqlContext.sql("select 1 as id, \"{\\\"animal\\\":{\\\"type\\\": \\\"cat\\\"}},{\\\"animal\\\":{\\\"type\\\": \\\"dog\\\"}},{\\\"animal\\\":{\\\"type\\\": \\\"donkey\\\"}},{\\\"animal\\\":{\\\"type\\\": \\\"turkey\\\"}},{\\\"animal\\\":{\\\"type\\\": \\\"cat\\\"}},{\\\"animal\\\":{\\\"NOTANIMAL\\\": \\\"measuring tape\\\"}}\" as field")
case class SubField(fieldling: String)
var data02 = data01.explode(data01("field")){ case Row(field: String) => field.split(",").map(SubField(_))}
.selectExpr("id","fieldling","get_json_object(fieldling,\"$.animal.type\") as animal")
var data03 = data01.explode(data01("field")){ case Row(field: String) => field.split(",").map(SubField(_))}
.selectExpr("id","fieldling","get_json_object(fieldling,\"$.animal.type\") as animal")
data02.cache()
data02.select($"animal" === "cat").explain
== Physical Plan ==
Project [(animal#25 = cat) AS (animal = cat)#263]
InMemoryColumnarTableScan [animal#25], (InMemoryRelation [id#20,fieldling#24,animal#25], true, 10000, StorageLevel(true, true, false, true, 1), (TungstenProject [id#20,fieldling#24,get_json_object(fieldling#24,$.animal.type) AS animal#25]), None)
data02.select($"animal" === "cat").show
+--------------+
|(animal = cat)|
+--------------+
| true|
| false|
| false|
| false|
| true|
| null|
+--------------+
data02.filter($"animal" === "cat").explain
== Physical Plan ==
Filter (animal#25 = cat)
InMemoryColumnarTableScan [id#20,fieldling#24,animal#25], [(animal#25 = cat)], (InMemoryRelation [id#20,fieldling#24,animal#25], true, 10000, StorageLevel(true, true, false, true, 1), (TungstenProject [id#20,fieldling#24,get_json_object(fieldling#24,$.animal.type) AS animal#25]), None)
data02.filter($"animal" === "cat").show
+---+---------+------+
| id|fieldling|animal|
+---+---------+------+
+---+---------+------+
{code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org