[jira] [Updated] (IGNITE-12435) [Spark] Add support for saving to existing table via saveAsTable

["Alexey Zinoviev (Jira)" <ji...@apache.org>]

     [ https://issues.apache.org/jira/browse/IGNITE-12435?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Alexey Zinoviev updated IGNITE-12435:
-------------------------------------
    Fix Version/s:     (was: 2.9)
                   3.0

> [Spark] Add support for saving to existing table via saveAsTable
> ----------------------------------------------------------------
>
>                 Key: IGNITE-12435
>                 URL: https://issues.apache.org/jira/browse/IGNITE-12435
>             Project: Ignite
>          Issue Type: Sub-task
>          Components: spark
>            Reporter: Alexey Zinoviev
>            Assignee: Alexey Zinoviev
>            Priority: Major
>             Fix For: 3.0
>
>
> Tests in IgniteSQLDataFrameIgniteSessionWriteSpec are muted due to strange error related to working with filesystems and schemas
>  
> All three tests generates the same error when you are trying to call saveAsTable as a terminal operation on dataframe write: 
> java.io.IOException: No FileSystem for scheme: ignitejava.io.IOException: No FileSystem for scheme: ignite
>  at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2586) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2593) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2632) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2614) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at org.apache.hadoop.fs.Path.getFileSystem(Path.java:296) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.validateTableLocation(SessionCatalog.scala:333) at org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand.run(createDataSourceTables.scala:170) at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:104) at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:102) at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:122) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80) at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676) at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676) at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78) at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73) at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676) at org.apache.spark.sql.DataFrameWriter.createTable(DataFrameWriter.scala:474) at org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:449) at org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:409) at org.apache.ignite.spark.IgniteSQLDataFrameIgniteSessionWriteSpec$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(IgniteSQLDataFrameIgniteSessionWriteSpec.scala:45) at org.apache.ignite.spark.IgniteSQLDataFrameIgniteSessionWriteSpec$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(IgniteSQLDataFrameIgniteSessionWriteSpec.scala:35) at org.apache.ignite.spark.IgniteSQLDataFrameIgniteSessionWriteSpec$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(IgniteSQLDataFrameIgniteSessionWriteSpec.scala:35) at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) at org.scalatest.Transformer.apply(Transformer.scala:22) at org.scalatest.Transformer.apply(Transformer.scala:20) at org.scalatest.FunSpecLike$$anon$1.apply(FunSpecLike.scala:422) at org.scalatest.Suite$class.withFixture(Suite.scala:1122) at org.scalatest.FunSpec.withFixture(FunSpec.scala:1626) at org.scalatest.FunSpecLike$class.invokeWithFixture$1(FunSpecLike.scala:419) at org.scalatest.FunSpecLike$$anonfun$runTest$1.apply(FunSpecLike.scala:431) at org.scalatest.FunSpecLike$$anonfun$runTest$1.apply(FunSpecLike.scala:431) at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) at org.scalatest.FunSpecLike$class.runTest(FunSpecLike.scala:431) at org.apache.ignite.spark.AbstractDataFrameSpec.org$scalatest$BeforeAndAfter$$super$runTest(AbstractDataFrameSpec.scala:39) at org.scalatest.BeforeAndAfter$class.runTest(BeforeAndAfter.scala:200) at org.apache.ignite.spark.AbstractDataFrameSpec.runTest(AbstractDataFrameSpec.scala:39) at org.scalatest.FunSpecLike$$anonfun$runTests$1.apply(FunSpecLike.scala:464) at org.scalatest.FunSpecLike$$anonfun$runTests$1.apply(FunSpecLike.scala:464) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) at scala.collection.immutable.List.foreach(List.scala:392) at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) at scala.collection.immutable.List.foreach(List.scala:392) at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) at org.scalatest.FunSpecLike$class.runTests(FunSpecLike.scala:464) at org.scalatest.FunSpec.runTests(FunSpec.scala:1626) at org.scalatest.Suite$class.run(Suite.scala:1424) at org.scalatest.FunSpec.org$scalatest$FunSpecLike$$super$run(FunSpec.scala:1626) at org.scalatest.FunSpecLike$$anonfun$run$1.apply(FunSpecLike.scala:468) at org.scalatest.FunSpecLike$$anonfun$run$1.apply(FunSpecLike.scala:468) at org.scalatest.SuperEngine.runImpl(Engine.scala:545) at org.scalatest.FunSpecLike$class.run(FunSpecLike.scala:468) at org.apache.ignite.spark.AbstractDataFrameSpec.org$scalatest$BeforeAndAfterAll$$super$run(AbstractDataFrameSpec.scala:39) at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257) at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) at org.apache.ignite.spark.AbstractDataFrameSpec.org$scalatest$BeforeAndAfter$$super$run(AbstractDataFrameSpec.scala:39) at org.scalatest.BeforeAndAfter$class.run(BeforeAndAfter.scala:241) at org.apache.ignite.spark.AbstractDataFrameSpec.run(AbstractDataFrameSpec.scala:39) at
>  
>  
>  
>  
>  
>  
>  
>  
> org.scalatest.junit.JUnitRunner.run(JUnitRunner.scala:99) at org.junit.runner.JUnitCore.run(JUnitCore.java:160) at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68) at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33) at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230) at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)



--
This message was sent by Atlassian Jira
(v8.3.4#803005)