This is an automated email from the ASF dual-hosted git repository.
fchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new cdbe05fa4 [KYUUBI #4720] [ARROW] Fix java.lang.NoSuchFieldError: IpcOption.DEFAULT for Spark-3.1/3.2
cdbe05fa4 is described below
commit cdbe05fa4c6a18cd52185f3152b8648f676b240d
Author: Fu Chen <cf...@gmail.com>
AuthorDate: Mon Apr 17 16:52:54 2023 +0800
[KYUUBI #4720] [ARROW] Fix java.lang.NoSuchFieldError: IpcOption.DEFAULT for Spark-3.1/3.2
### _Why are the changes needed?_
`IpcOption.DEFAULT` was introduced in [ARROW-11081](https://github.com/apache/arrow/pull/9053)(ARROW-4.0.0), add `ARROW_IPC_OPTION_DEFAULT` for adapt Spark-3.1/3.2
```
Caused by: java.lang.NoSuchFieldError: DEFAULT
at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.$anonfun$next$1(KyuubiArrowConverters.scala:304)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1491)
at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.next(KyuubiArrowConverters.scala:308)
at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.next(KyuubiArrowConverters.scala:231)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at org.apache.spark.sql.execution.arrow.KyuubiArrowConverters$ArrowBatchIterator.foreach(KyuubiArrowConverters.scala:231)
```
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4720 from cfmcgrady/arrow-ipc-option.
Closes #4720
2c80e670e [Fu Chen] fix style
a8294f637 [Fu Chen] add ARROW_IPC_OPTION_DEFAULT
Authored-by: Fu Chen <cf...@gmail.com>
Signed-off-by: Fu Chen <cf...@gmail.com>
---
.../org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala
index dd6163ec9..2feadbced 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala
@@ -299,7 +299,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
MessageSerializer.serialize(writeChannel, batch)
// Always write the Ipc options at the end.
- ArrowStreamWriter.writeEndOfStream(writeChannel, IpcOption.DEFAULT)
+ ArrowStreamWriter.writeEndOfStream(writeChannel, ARROW_IPC_OPTION_DEFAULT)
batch.close()
} {
@@ -318,4 +318,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
context: TaskContext): Iterator[InternalRow] = {
ArrowConverters.fromBatchIterator(arrowBatchIter, schema, timeZoneId, context)
}
+
+ // IpcOption.DEFAULT was introduced in ARROW-11081(ARROW-4.0.0), add this for adapt Spark-3.1/3.2
+ final private val ARROW_IPC_OPTION_DEFAULT = new IpcOption()
}