You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Yana Kadiyska <ya...@gmail.com> on 2014/10/24 20:32:56 UTC
[Spark SQL] Setting variables
Hi all,
I'm trying to set a pool for a JDBC session. I'm connecting to the thrift
server via JDBC client.
My installation appears to be good(queries run fine), I can see the pools
in the UI, but any attempt to set a variable (I tried
spark.sql.shuffle.partitions and spark.sql.thriftserver.scheduler.pool)
result in the exception below (trace is from Thriftserver log)
Any thoughts on what I'm doing wrong? (I am on master, built today)
SET spark.sql.thriftserver.scheduler.pool=mypool;select count(*) from mytable;
======================
14/10/24 18:17:10 ERROR server.SparkSQLOperationManager: Error executing
query:
java.lang.NullPointerException
at
org.apache.spark.sql.hive.HiveContext.runHive(HiveContext.scala:309)
at
org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:272)
at
org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:244)
at
org.apache.spark.sql.execution.SetCommand.sideEffectResult$lzycompute(commands.scala:64)
at
org.apache.spark.sql.execution.SetCommand.sideEffectResult(commands.scala:55)
at
org.apache.spark.sql.execution.Command$class.execute(commands.scala:44)
at
org.apache.spark.sql.execution.SetCommand.execute(commands.scala:51)
at
org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd$lzycompute(HiveContext.scala:357)
at
org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd(HiveContext.scala:357)
at
org.apache.spark.sql.SchemaRDDLike$class.$init$(SchemaRDDLike.scala:58)
at org.apache.spark.sql.SchemaRDD.<init>(SchemaRDD.scala:104)
at org.apache.spark.sql.hive.HiveContext.sql(HiveContext.scala:99)
at
org.apache.spark.sql.hive.thriftserver.server.SparkSQLOperationManager$$anon$1.run(SparkSQLOperationManager.scala:172)
at
org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:193)
at
org.apache.hive.service.cli.session.HiveSessionImpl.executeStatement(HiveSessionImpl.java:175)
at
org.apache.hive.service.cli.CLIService.executeStatement(CLIService.java:150)
at
org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:207)
at
org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1133)
at
org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1118)
at
org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at
org.apache.hive.service.auth.TUGIContainingProcessor$1.run(TUGIContainingProcessor.java:58)
at
org.apache.hive.service.auth.TUGIContainingProcessor$1.run(TUGIContainingProcessor.java:55)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
at
org.apache.hadoop.hive.shims.HadoopShimsSecure.doAs(HadoopShimsSecure.java:526)
at
org.apache.hive.service.auth.TUGIContainingProcessor.process(TUGIContainingProcessor.java:55)
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:206)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Re: [Spark SQL] Setting variables
Posted by Michael Armbrust <mi...@databricks.com>.
You might be hitting: https://issues.apache.org/jira/browse/SPARK-4037
On Fri, Oct 24, 2014 at 11:32 AM, Yana Kadiyska <ya...@gmail.com>
wrote:
> Hi all,
>
> I'm trying to set a pool for a JDBC session. I'm connecting to the thrift
> server via JDBC client.
>
> My installation appears to be good(queries run fine), I can see the pools
> in the UI, but any attempt to set a variable (I tried
> spark.sql.shuffle.partitions and spark.sql.thriftserver.scheduler.pool)
> result in the exception below (trace is from Thriftserver log)
>
>
> Any thoughts on what I'm doing wrong? (I am on master, built today)
>
> SET spark.sql.thriftserver.scheduler.pool=mypool;select count(*) from mytable;
>
>
> ======================
>
> 14/10/24 18:17:10 ERROR server.SparkSQLOperationManager: Error executing
> query:
> java.lang.NullPointerException
> at
> org.apache.spark.sql.hive.HiveContext.runHive(HiveContext.scala:309)
> at
> org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:272)
> at
> org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:244)
> at
> org.apache.spark.sql.execution.SetCommand.sideEffectResult$lzycompute(commands.scala:64)
> at
> org.apache.spark.sql.execution.SetCommand.sideEffectResult(commands.scala:55)
> at
> org.apache.spark.sql.execution.Command$class.execute(commands.scala:44)
> at
> org.apache.spark.sql.execution.SetCommand.execute(commands.scala:51)
> at
> org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd$lzycompute(HiveContext.scala:357)
> at
> org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd(HiveContext.scala:357)
> at
> org.apache.spark.sql.SchemaRDDLike$class.$init$(SchemaRDDLike.scala:58)
> at org.apache.spark.sql.SchemaRDD.<init>(SchemaRDD.scala:104)
> at org.apache.spark.sql.hive.HiveContext.sql(HiveContext.scala:99)
> at
> org.apache.spark.sql.hive.thriftserver.server.SparkSQLOperationManager$$anon$1.run(SparkSQLOperationManager.scala:172)
> at
> org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:193)
> at
> org.apache.hive.service.cli.session.HiveSessionImpl.executeStatement(HiveSessionImpl.java:175)
> at
> org.apache.hive.service.cli.CLIService.executeStatement(CLIService.java:150)
> at
> org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:207)
> at
> org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1133)
> at
> org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1118)
> at
> org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
> at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
> at
> org.apache.hive.service.auth.TUGIContainingProcessor$1.run(TUGIContainingProcessor.java:58)
> at
> org.apache.hive.service.auth.TUGIContainingProcessor$1.run(TUGIContainingProcessor.java:55)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
> at
> org.apache.hadoop.hive.shims.HadoopShimsSecure.doAs(HadoopShimsSecure.java:526)
> at
> org.apache.hive.service.auth.TUGIContainingProcessor.process(TUGIContainingProcessor.java:55)
> at
> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:206)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:744)
>
>
>