You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by fe...@apache.org on 2024/04/26 19:16:54 UTC
(incubator-gluten) branch main updated: [VL] [BUG fix] Make the hasNext method can be called multi times (#5545)
This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0e4f87014 [VL] [BUG fix] Make the hasNext method can be called multi times (#5545)
0e4f87014 is described below
commit 0e4f870145ec414c3abb227c3175d6edb5dbcf39
Author: JiaKe <ke...@intel.com>
AuthorDate: Sat Apr 27 03:16:49 2024 +0800
[VL] [BUG fix] Make the hasNext method can be called multi times (#5545)
Currently, the iterator is designed to close once the hasNext() method returns false. Therefore, if hasNext() is called a second time, it will throw exception here. This PR adds a protectInvocationFlow() method to ensure that the iterator's hasNext() can be safely called multiple times without causing an exception.
---
.../scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala | 1 +
.../scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala | 1 +
.../org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala | 1 +
cpp/core/jni/JniWrapper.cc | 3 ++-
.../org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala | 1 +
.../src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala | 1 +
6 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
index 5b0918976..5a165116c 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
@@ -225,6 +225,7 @@ class VeloxIteratorApi extends IteratorApi with Logging {
Iterators
.wrap(nativeResultIterator.asScala)
+ .protectInvocationFlow()
.recycleIterator {
updateNativeMetrics(nativeResultIterator.getMetrics)
nativeResultIterator.close()
diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
index 32358b079..22ca020d1 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
@@ -231,6 +231,7 @@ object RowToVeloxColumnarExec {
}
Iterators
.wrap(res)
+ .protectInvocationFlow()
.recycleIterator {
jniWrapper.close(r2cHandle)
}
diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
index ce210b3bf..88678cb5e 100644
--- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
+++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
@@ -272,6 +272,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
}
}
})
+ .protectInvocationFlow()
.recycleIterator {
jniWrapper.close(deserializerHandle)
}
diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc
index 748eb5831..7363a9da0 100644
--- a/cpp/core/jni/JniWrapper.cc
+++ b/cpp/core/jni/JniWrapper.cc
@@ -449,7 +449,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIte
auto iter = ctx->objectStore()->retrieve<ResultIterator>(iterHandle);
if (iter == nullptr) {
- std::string errorMessage = "failed to get batch iterator";
+ std::string errorMessage =
+ "When hasNext() is called on a closed iterator, an exception is thrown. To prevent this, consider using the protectInvocationFlow() method when creating the iterator in scala side. This will allow the hasNext() method to be called multiple times without issue.";
throw gluten::GlutenException(errorMessage);
}
return iter->hasNext();
diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
index c3e39d834..ac94bfa89 100644
--- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
+++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
@@ -74,6 +74,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
ColumnarBatches.create(Runtimes.contextInstance(), handle)
}
})
+ .protectInvocationFlow()
.recycleIterator {
jniWrapper.close(serializeHandle)
}
diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
index 25a2938a8..fd0cf45c8 100644
--- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
+++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
@@ -67,6 +67,7 @@ object ExecUtil {
row
}
})
+ .protectInvocationFlow()
.recycleIterator {
jniWrapper.nativeClose(c2rHandle)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org