You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/09/18 01:10:51 UTC
(asterixdb) 01/02: [ASTERIXDB-3847][RT] Fix UDF argument string encoding
This is an automated email from the ASF dual-hosted git repository.
parshimers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit cae1fbd02bbd3d4ee2360b4dfa5f3f6189f172f9
Author: Ian Maxon <im...@apache.org>
AuthorDate: Tue Sep 15 15:21:24 2026 +0000
[ASTERIXDB-3847][RT] Fix UDF argument string encoding
Details:
The length of a input string was being calculated as
modified UTF-8 but written as normal UTF-8. Therefore
if you have emojis or other weird characters the length
will be too long. That can be OK for single values,
but if it's in the middle of a record you start reading
at the wrong offsets.
Generated-By: Claude Opus 5 <no...@anthropic.com>
Change-Id: I30b191a3767e437f110afc70176fd8e1233c1a29
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21797
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../src/test/resources/TweetSent/roundtrip.py | 3 +
.../external-library/py_utf8/py_utf8.0.ddl.sqlpp | 20 ++++
.../external-library/py_utf8/py_utf8.1.lib.sqlpp | 19 ++++
.../external-library/py_utf8/py_utf8.2.ddl.sqlpp | 28 +++++
.../external-library/py_utf8/py_utf8.3.query.sqlpp | 27 +++++
.../external-library/py_utf8/py_utf8.4.query.sqlpp | 26 +++++
.../external-library/py_utf8/py_utf8.5.query.sqlpp | 27 +++++
.../external-library/py_utf8/py_utf8.6.query.sqlpp | 26 +++++
.../external-library/py_utf8/py_utf8.7.query.sqlpp | 27 +++++
.../external-library/py_utf8/py_utf8.8.ddl.sqlpp | 19 ++++
.../results/external-library/py_utf8/py_utf8.3.adm | 1 +
.../results/external-library/py_utf8/py_utf8.4.adm | 1 +
.../results/external-library/py_utf8/py_utf8.5.adm | 1 +
.../results/external-library/py_utf8/py_utf8.6.adm | 1 +
.../results/external-library/py_utf8/py_utf8.7.adm | 1 +
.../resources/runtimets/testsuite_it_python.xml | 5 +
.../external/library/msgpack/MsgPackAccessors.java | 7 +-
.../library/msgpack/MsgPackStringAccessorTest.java | 120 +++++++++++++++++++++
.../data/nontagged/printers/PrintTools.java | 38 +++++++
19 files changed, 395 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py b/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
index 0d52f01dc6..852a7d4511 100644
--- a/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
+++ b/asterixdb/asterix-app/src/test/resources/TweetSent/roundtrip.py
@@ -28,6 +28,9 @@ class Tests(object):
def roundstr(self, arg):
return arg
+ def codepoints(self, arg):
+ return [ord(c) for c in arg]
+
def warning(self):
raise ArithmeticError("oof")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.0.ddl.sqlpp
new file mode 100644
index 0000000000..7c71deba49
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.0.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.1.lib.sqlpp
new file mode 100644
index 0000000000..699e5652e9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.1.lib.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+install externallibtest testlib python admin admin target/TweetSent.pyz
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.2.ddl.sqlpp
new file mode 100644
index 0000000000..6be70bcbf3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.2.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+create function roundstr(s)
+ as "roundtrip", "Tests.roundstr" at testlib;
+
+create function roundtrip2(a, b)
+ as "roundtrip", "Tests.roundtrip" at testlib;
+
+create function codepoints(s)
+ as "roundtrip", "Tests.codepoints" at testlib;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.3.query.sqlpp
new file mode 100644
index 0000000000..1a8f23a4ef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : A supplementary character is stored as a modified-UTF8 surrogate pair of 6 bytes but msgpack
+ * expects the 4 bytes of standard UTF-8, so check Python sees the codepoints we sent.
+ * Expected Res : Success
+ */
+
+use externallibtest;
+
+codepoints(codepoint_to_string([104, 128512, 105, 128169, 233, 20320]));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.4.query.sqlpp
new file mode 100644
index 0000000000..7b00cf0db4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Round trip a string of supplementary characters back out of Python.
+ * Expected Res : Success
+ */
+
+use externallibtest;
+
+string_to_codepoint(roundstr(codepoint_to_string([128512, 97, 128169])));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.5.query.sqlpp
new file mode 100644
index 0000000000..f4e30a8b75
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.5.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : A string header that overstates its payload does not corrupt only that string: everything packed
+ * after it in the same message is read at the wrong offset. Check the argument that follows one.
+ * Expected Res : Success
+ */
+
+use externallibtest;
+
+roundtrip2(codepoint_to_string([128512]), "tail")[1];
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.6.query.sqlpp
new file mode 100644
index 0000000000..adae804ce2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.6.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : The same, within a record: the field after a supplementary character must still arrive intact.
+ * Expected Res : Success
+ */
+
+use externallibtest;
+
+roundstr({"s": codepoint_to_string([128512, 32, 128169]), "n": 42}).n;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.7.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.7.query.sqlpp
new file mode 100644
index 0000000000..aa004bd067
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.7.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Supplementary characters interleaved with 1, 2 and 3 byte ones.
+ * Expected Res : Success
+ */
+
+use externallibtest;
+
+string_to_codepoint(roundstr("a" || codepoint_to_string([128512]) || "b" || codepoint_to_string([233])
+ || "c" || codepoint_to_string([20320]) || "d"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.8.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.8.ddl.sqlpp
new file mode 100644
index 0000000000..65733e4ffd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/py_utf8/py_utf8.8.ddl.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.3.adm
new file mode 100644
index 0000000000..b901afe593
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.3.adm
@@ -0,0 +1 @@
+[ 104, 128512, 105, 128169, 233, 20320 ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.4.adm
new file mode 100644
index 0000000000..a02db3c17a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.4.adm
@@ -0,0 +1 @@
+[ 128512, 97, 128169 ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.5.adm
new file mode 100644
index 0000000000..531cb0ba79
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.5.adm
@@ -0,0 +1 @@
+"tail"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.6.adm
new file mode 100644
index 0000000000..d81cc0710e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.6.adm
@@ -0,0 +1 @@
+42
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.7.adm
new file mode 100644
index 0000000000..69f99a4580
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/py_utf8/py_utf8.7.adm
@@ -0,0 +1 @@
+[ 97, 128512, 98, 233, 99, 20320, 100 ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index f65ebf203d..1702a0c13d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -45,6 +45,11 @@
<output-dir compare="Clean-JSON">py_nested_access</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-library">
+ <compilation-unit name="py_utf8">
+ <output-dir compare="Clean-JSON">py_utf8</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-library">
<compilation-unit name="python_open_type_validation">
<output-dir compare="Clean-JSON">python_open_type_validation</output-dir>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java
index 6f097d09e3..d3d7c1af5a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/msgpack/MsgPackAccessors.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.external.library.msgpack;
-import static org.apache.hyracks.util.string.UTF8StringUtil.getUTFLength;
import static org.msgpack.core.MessagePack.Code.ARRAY32;
import static org.msgpack.core.MessagePack.Code.FALSE;
import static org.msgpack.core.MessagePack.Code.FLOAT32;
@@ -59,6 +58,7 @@ import org.apache.asterix.om.types.TypeTagUtil;
import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
public class MsgPackAccessors {
@@ -172,11 +172,14 @@ public class MsgPackAccessors {
}
public static class MsgPackStringAccessor {
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Declare the UTF-8 length rather than the stored modified-UTF8 length")
public static Void apply(IPointable pointable, DataOutput out) throws IOException {
byte[] b = pointable.getByteArray();
int s = pointable.getStartOffset();
out.writeByte(STR32);
- final int calculatedLength = getUTFLength(b, s + 1);
+ // msgpack strings are standard UTF-8, in which a supplementary character occupies 4 bytes rather than
+ // the 6 it takes as a modified-UTF8 surrogate pair, so the stored length would overstate the payload
+ final int calculatedLength = PrintTools.getUTF8StringRawLength(b, s + 1);
out.writeInt(calculatedLength);
PrintTools.writeUTF8StringRaw(b, s + 1, calculatedLength, out);
return null;
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/msgpack/MsgPackStringAccessorTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/msgpack/MsgPackStringAccessorTest.java
new file mode 100644
index 0000000000..d4c2775267
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/msgpack/MsgPackStringAccessorTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.library.msgpack;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+import org.junit.Test;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessageUnpacker;
+
+/**
+ * Strings handed to a Python UDF must be packed as standard UTF-8, not as the modified UTF-8 they are stored in.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Regression test for supplementary characters mis-sized in msgpack STR headers")
+public class MsgPackStringAccessorTest {
+
+ private static final String EMOJI = "😀";
+
+ @Test
+ public void testAscii() throws Exception {
+ assertPacksAs("hello");
+ }
+
+ @Test
+ public void testTwoByteChars() throws Exception {
+ assertPacksAs("café über");
+ }
+
+ @Test
+ public void testThreeByteChars() throws Exception {
+ assertPacksAs("ä½ å¥½ä¸–ç•Œ");
+ }
+
+ @Test
+ public void testSupplementaryChars() throws Exception {
+ assertPacksAs(EMOJI);
+ assertPacksAs("a" + EMOJI + "b");
+ assertPacksAs(EMOJI + EMOJI + EMOJI);
+ assertPacksAs("mixed é ä½ " + EMOJI + " tail");
+ }
+
+ /**
+ * A STR header that overstates its payload does not corrupt only that string: the unpacker keeps reading into
+ * whatever follows it in the message.
+ */
+ @Test
+ public void testSupplementaryCharsDoNotDesyncFollowingValues() throws Exception {
+ ArrayBackedValueStorage packed = new ArrayBackedValueStorage();
+ pack("a" + EMOJI, packed);
+ pack("b" + EMOJI, packed);
+ try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(packed.getByteArray(), 0, packed.getLength())) {
+ assertEquals("a" + EMOJI, unpacker.unpackString());
+ assertEquals("b" + EMOJI, unpacker.unpackString());
+ assertFalse(unpacker.hasNext());
+ }
+ }
+
+ /**
+ * Malformed input still has to be framed correctly: a STR header disagreeing with its payload would leave the
+ * rest of the message unreadable, which is a worse failure than the mojibake the character itself becomes.
+ */
+ @Test
+ public void testUnpairedHighSurrogateIsStillFramedCorrectly() throws Exception {
+ ArrayBackedValueStorage packed = new ArrayBackedValueStorage();
+ pack("\uD800a", packed);
+ pack("tail", packed);
+ try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(packed.getByteArray(), 0, packed.getLength())) {
+ unpacker.unpackString();
+ assertEquals("tail", unpacker.unpackString());
+ assertFalse(unpacker.hasNext());
+ }
+ }
+
+ private static void assertPacksAs(String value) throws Exception {
+ ArrayBackedValueStorage packed = new ArrayBackedValueStorage();
+ pack(value, packed);
+ try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(packed.getByteArray(), 0, packed.getLength())) {
+ assertEquals(value, unpacker.unpackString());
+ assertFalse(unpacker.hasNext());
+ }
+ }
+
+ private static void pack(String value, ArrayBackedValueStorage out) throws Exception {
+ ArrayBackedValueStorage serialized = new ArrayBackedValueStorage();
+ DataOutput serializedOut = serialized.getDataOutput();
+ serializedOut.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG);
+ new AStringSerializerDeserializer(new UTF8StringWriter(), new UTF8StringReader()).serialize(new AString(value),
+ serializedOut);
+ VoidPointable pointable = new VoidPointable();
+ pointable.set(serialized.getByteArray(), 0, serialized.getLength());
+ MsgPackAccessors.createFlatMsgPackAccessor(ATypeTag.STRING).apply(pointable, out.getDataOutput());
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java
index cb6dfb98f8..6fc105304d 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java
@@ -36,6 +36,7 @@ import org.apache.asterix.dataflow.data.nontagged.serde.jacksonjts.JtsModule;
import org.apache.asterix.om.base.temporal.GregorianCalendarSystem;
import org.apache.hyracks.algebricks.data.utils.WriteValueTools;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.apache.hyracks.util.bytes.HexPrinter;
import org.apache.hyracks.util.string.UTF8StringUtil;
import org.locationtech.jts.geom.Geometry;
@@ -353,6 +354,43 @@ public class PrintTools {
}
}
+ /**
+ * Computes the number of bytes {@link #writeUTF8StringRaw} will emit for the string stored at {@code s}. This is
+ * not the stored length: a supplementary character is stored as a surrogate pair of two 3-byte modified-UTF8
+ * characters, but is written out as a single 4-byte UTF-8 sequence.
+ *
+ * @param b the buffer holding a length-prefixed modified-UTF8 string
+ * @param s the offset of that string's length prefix
+ * @return the length, in bytes, of the standard UTF-8 encoding of that string
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "UTF-8 length companion to writeUTF8StringRaw")
+ public static int getUTF8StringRawLength(byte[] b, int s) {
+ final int utfLength = UTF8StringUtil.getUTFLength(b, s);
+ int position = s + UTF8StringUtil.getNumBytesToStoreLength(utfLength);
+ final int maxPosition = position + utfLength;
+ int rawLength = 0;
+ while (position < maxPosition) {
+ final int sz = UTF8StringUtil.charSize(b, position);
+ final char c = UTF8StringUtil.charAt(b, position);
+ if (!Character.isHighSurrogate(c)) {
+ rawLength += sz;
+ position += sz;
+ continue;
+ }
+ final int lowSurrogatePos = position + sz;
+ if (lowSurrogatePos >= maxPosition) {
+ throw new IllegalStateException("malformed utf8 input");
+ }
+ final char lowSurrogate = UTF8StringUtil.charAt(b, lowSurrogatePos);
+ // the malformed case is spelled the same way writeSupplementaryChar encodes it, so the two cannot
+ // disagree on a length that must already have been written into a msgpack header
+ rawLength += Character.isLowSurrogate(lowSurrogate) ? 4
+ : new String(new char[] { c, lowSurrogate }).getBytes(StandardCharsets.UTF_8).length;
+ position = lowSurrogatePos + UTF8StringUtil.charSize(b, lowSurrogatePos);
+ }
+ return rawLength;
+ }
+
public static void writeUTF8StringRaw(byte[] b, int s, int l, DataOutput os) throws IOException {
int utfLength = UTF8StringUtil.getUTFLength(b, s);
int position = s + UTF8StringUtil.getNumBytesToStoreLength(utfLength); // skip 2 bytes containing string size