You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Yi Zhou (JIRA)" <ji...@apache.org> on 2015/07/27 09:50:04 UTC
[jira] [Updated] (SPARK-9374) [Spark SQL] UDFUnixTimeStamp throw
out erorr of "AnalysisException: nondeterministic expressions are only
allowed in Project or Filter" during the spark sql parse phase
[ https://issues.apache.org/jira/browse/SPARK-9374?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yi Zhou updated SPARK-9374:
---------------------------
Summary: [Spark SQL] UDFUnixTimeStamp throw out erorr of "AnalysisException: nondeterministic expressions are only allowed in Project or Filter" during the spark sql parse phase (was: [Spark SQL] Throw out erorr of "AnalysisException: nondeterministic expressions are only allowed in Project or Filter" during the spark sql parse phase)
> [Spark SQL] UDFUnixTimeStamp throw out erorr of "AnalysisException: nondeterministic expressions are only allowed in Project or Filter" during the spark sql parse phase
> ------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: SPARK-9374
> URL: https://issues.apache.org/jira/browse/SPARK-9374
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 1.5.0
> Reporter: Yi Zhou
> Priority: Blocker
>
> #################Spark SQL Query########
> INSERT INTO TABLE TEST_QUERY_0_result
> SELECT w_state, i_item_id,
> SUM(
> CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') < unix_timestamp('2001-03-16','yyyy-MM-dd'))
> THEN ws_sales_price - COALESCE(wr_refunded_cash,0)
> ELSE 0.0 END
> ) AS sales_before,
> SUM(
> CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') >= unix_timestamp('2001-03-16','yyyy-MM-dd'))
> THEN ws_sales_price - coalesce(wr_refunded_cash,0)
> ELSE 0.0 END
> ) AS sales_after
> FROM (
> SELECT *
> FROM web_sales ws
> LEFT OUTER JOIN web_returns wr ON (ws.ws_order_number = wr.wr_order_number
> AND ws.ws_item_sk = wr.wr_item_sk)
> ) a1
> JOIN item i ON a1.ws_item_sk = i.i_item_sk
> JOIN warehouse w ON a1.ws_warehouse_sk = w.w_warehouse_sk
> JOIN date_dim d ON a1.ws_sold_date_sk = d.d_date_sk
> AND unix_timestamp(d.d_date, 'yyyy-MM-dd') >= unix_timestamp('2001-03-16', 'yyyy-MM-dd') - 30*24*60*60 --subtract 30 days in seconds
> AND unix_timestamp(d.d_date, 'yyyy-MM-dd') <= unix_timestamp('2001-03-16', 'yyyy-MM-dd') + 30*24*60*60 --add 30 days in seconds
> GROUP BY w_state,i_item_id
> CLUSTER BY w_state,i_item_id
> ############Error Message##################
> org.apache.spark.sql.AnalysisException: nondeterministic expressions are only allowed in Project or Filter, found:
> (((ws_sold_date_sk = d_date_sk) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd) >= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) - CAST((((30 * 24) * 60) * 60), LongType)))) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd) <= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) + CAST((((30 * 24) * 60) * 60), LongType))))
> in operator Join Inner, Some((((ws_sold_date_sk#289L = d_date_sk#383L) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd) >= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) - CAST((((30 * 24) * 60) * 60), LongType)))) && (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd) <= (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) + CAST((((30 * 24) * 60) * 60), LongType)))))
> ;
> at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37)
> at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:43)
> at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:148)
> at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102)
> at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49)
> at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:43)
> at org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:976)
> at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:131)
> at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:52)
> at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:792)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:61)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:284)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:423)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359)
> at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:456)
> at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:466)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:163)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:666)
> at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:178)
> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:203)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:118)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org