You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/09/06 07:18:05 UTC
[incubator-paimon] branch master updated: [Bug] Fix spark array and map type error when write (#1932)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fd3ae1ff9 [Bug] Fix spark array and map type error when write (#1932)
fd3ae1ff9 is described below
commit fd3ae1ff92935258ad0163ceb1a6ea782725b5a1
Author: leoyy0316 <10...@users.noreply.github.com>
AuthorDate: Wed Sep 6 15:18:00 2023 +0800
[Bug] Fix spark array and map type error when write (#1932)
---
.../java/org/apache/paimon/spark/SparkRow.java | 23 ++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
index ecd1472ef..e4928974b 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
@@ -32,6 +32,8 @@ import org.apache.paimon.types.RowKind;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.DateTimeUtils;
+import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
+
import org.apache.spark.sql.Row;
import java.io.Serializable;
@@ -44,6 +46,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import scala.collection.JavaConverters;
+import scala.collection.mutable.WrappedArray;
+
/** A {@link InternalRow} wraps spark {@link Row}. */
public class SparkRow implements InternalRow, Serializable {
@@ -286,12 +291,26 @@ public class SparkRow implements InternalRow, Serializable {
@Override
public InternalArray getArray(int i) {
- return new PaimonArray(((ArrayType) elementType).getElementType(), getAs(i));
+ Object array = getAs(i);
+ if (array instanceof WrappedArray) {
+ List<Object> result = Lists.newArrayList();
+ ((WrappedArray) array).iterator().foreach(x -> result.add(x));
+ return new PaimonArray(((ArrayType) elementType).getElementType(), result);
+ }
+ return new PaimonArray(
+ ((ArrayType) elementType).getElementType(), (List<Object>) array);
}
@Override
public InternalMap getMap(int i) {
- return toPaimonMap((MapType) elementType, getAs(i));
+ Object map = getAs(i);
+ if (map instanceof scala.collection.immutable.Map) {
+ return toPaimonMap(
+ (MapType) elementType,
+ JavaConverters.mapAsJavaMap(
+ (scala.collection.immutable.Map<Object, Object>) map));
+ }
+ return toPaimonMap((MapType) elementType, (Map<Object, Object>) map);
}
@Override