You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pa...@apache.org on 2026/10/19 03:32:12 UTC
(arrow-nanoarrow) branch main updated: fix: Don't truncate metadata keys and values at NUL bytes (#971)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 10885c5f fix: Don't truncate metadata keys and values at NUL bytes (#971)
10885c5f is described below
commit 10885c5f19c45d85fecf00b7d7b169debe732207
Author: Rusty Conover <ru...@conover.me>
AuthorDate: Sun Oct 18 23:32:06 2026 -0400
fix: Don't truncate metadata keys and values at NUL bytes (#971)
Closes #970.
`ArrowIpcEncodeMetadata()` wrote `KeyValue` keys and values with
flatcc's `_create_strn()`. That function stops at the first NUL byte
(`flatcc_builder_create_string(B, s, strnlen(s, max_len))`), so a
metadata key or value containing a NUL was silently truncated in Schema,
Field and Message `custom_metadata`. NUL (U+0000) is valid UTF-8, and
both nanoarrow's metadata format and flatbuffers strings store a length
with each string, so the encoder now uses the length-exact `_create()`
to write exactly `size_bytes` bytes. The decoder already reads keys and
values by length. No other IPC encoder code uses `_create_strn()`.
The new `NanoarrowIpcEncoderMetadataWithNulBytes` test round-trips
schema, field and message metadata whose keys and values contain NUL
bytes, through the encoder/decoder and through
`ArrowIpcWriter`/`ArrowIpcArrayStreamReader`, and checks that the bytes
match exactly. It fails without the fix: `("field\0key",
"\0field\0value")` decoded as `("field", "")`. `ctest` passes with and
without Arrow C++, also under ASan/UBSan, and pre-commit passes on the
changed files.
This PR's code and description were written by AI (Claude Code) and
reviewed by a human maintainer.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Rusty Conover <ru...@query.farm>
Co-authored-by: Claude Opus 5.5 (1M context) <no...@anthropic.com>
---
src/nanoarrow/ipc/encoder.c | 5 +--
src/nanoarrow/ipc/encoder_test.cc | 86 +++++++++++++++++++++++++++++++++++++--
2 files changed, 85 insertions(+), 6 deletions(-)
diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c
index 2fe14713..de325c19 100644
--- a/src/nanoarrow/ipc/encoder.c
+++ b/src/nanoarrow/ipc/encoder.c
@@ -493,10 +493,9 @@ static ArrowErrorCode ArrowIpcEncodeMetadata(flatcc_builder_t* builder,
NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowMetadataReaderRead(&metadata, &key, &value),
error);
FLATCC_RETURN_UNLESS_0_NO_NS(push_start(builder), error);
- FLATCC_RETURN_UNLESS_0(KeyValue_key_create_strn(builder, key.data, key.size_bytes),
+ FLATCC_RETURN_UNLESS_0(KeyValue_key_create(builder, key.data, key.size_bytes), error);
+ FLATCC_RETURN_UNLESS_0(KeyValue_value_create(builder, value.data, value.size_bytes),
error);
- FLATCC_RETURN_UNLESS_0(
- KeyValue_value_create_strn(builder, value.data, value.size_bytes), error);
FLATCC_RETURN_IF_NULL(push_end(builder), error);
}
return NANOARROW_OK;
diff --git a/src/nanoarrow/ipc/encoder_test.cc b/src/nanoarrow/ipc/encoder_test.cc
index b6e7e4ac..54dd4547 100644
--- a/src/nanoarrow/ipc/encoder_test.cc
+++ b/src/nanoarrow/ipc/encoder_test.cc
@@ -152,9 +152,10 @@ static nanoarrow::UniqueBuffer PackMetadata(const KeyValues& key_values) {
nanoarrow::UniqueBuffer metadata;
NANOARROW_THROW_NOT_OK(ArrowMetadataBuilderInit(metadata.get(), nullptr));
for (const auto& kv : key_values) {
- NANOARROW_THROW_NOT_OK(ArrowMetadataBuilderAppend(metadata.get(),
- ArrowCharView(kv.first.c_str()),
- ArrowCharView(kv.second.c_str())));
+ struct ArrowStringView key = {kv.first.data(), static_cast<int64_t>(kv.first.size())};
+ struct ArrowStringView value = {kv.second.data(),
+ static_cast<int64_t>(kv.second.size())};
+ NANOARROW_THROW_NOT_OK(ArrowMetadataBuilderAppend(metadata.get(), key, value));
}
return metadata;
}
@@ -286,6 +287,85 @@ TEST(NanoarrowIpcTest, NanoarrowIpcEncoderMessageMetadataRoundtrip) {
EXPECT_EQ(DecodeMessageMetadata(message.get(), decoder.get()), KeyValues{});
}
+// Metadata keys and values are length-prefixed both in ArrowSchema and in the
+// flatbuffer, so they may contain NUL bytes (which are valid UTF-8) and must be
+// written exactly rather than truncated at the first NUL.
+TEST(NanoarrowIpcTest, NanoarrowIpcEncoderMetadataWithNulBytes) {
+ using namespace std::string_literals;
+ KeyValues schema_key_values{{"schema\0key"s, "schema\0value\0"s}, {"\0"s, ""s}};
+ KeyValues field_key_values{{"field\0key"s, "\0field\0value"s}};
+ KeyValues message_key_values{{"message\0key"s, "message\0value"s}};
+
+ nanoarrow::UniqueSchema schema;
+ ASSERT_EQ(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaAllocateChildren(schema.get(), 1), NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT32),
+ NANOARROW_OK);
+ ASSERT_EQ(ArrowSchemaSetName(schema->children[0], "col"), NANOARROW_OK);
+ auto schema_metadata = PackMetadata(schema_key_values);
+ ASSERT_EQ(ArrowSchemaSetMetadata(schema.get(),
+ reinterpret_cast<const char*>(schema_metadata->data)),
+ NANOARROW_OK);
+ auto field_metadata = PackMetadata(field_key_values);
+ ASSERT_EQ(ArrowSchemaSetMetadata(schema->children[0],
+ reinterpret_cast<const char*>(field_metadata->data)),
+ NANOARROW_OK);
+
+ // Encode and decode a Schema message
+ nanoarrow::ipc::UniqueEncoder encoder;
+ ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK);
+ struct ArrowError error;
+ auto message_metadata = PackMetadata(message_key_values);
+ ASSERT_EQ(
+ ArrowIpcEncoderSetMessageMetadata(encoder.get(), message_metadata.get(), &error),
+ NANOARROW_OK)
+ << error.message;
+ ASSERT_EQ(ArrowIpcEncoderEncodeSchema(encoder.get(), schema.get(), &error),
+ NANOARROW_OK)
+ << error.message;
+ nanoarrow::UniqueBuffer message;
+ ASSERT_EQ(
+ ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, message.get()),
+ NANOARROW_OK);
+
+ nanoarrow::ipc::UniqueDecoder decoder;
+ ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK);
+ EXPECT_EQ(DecodeMessageMetadata(message.get(), decoder.get()), message_key_values);
+
+ nanoarrow::UniqueSchema decoded;
+ ASSERT_EQ(ArrowIpcDecoderDecodeSchema(decoder.get(), decoded.get(), &error),
+ NANOARROW_OK)
+ << error.message;
+ EXPECT_EQ(UnpackMetadata(decoded->metadata), schema_key_values);
+ ASSERT_EQ(decoded->n_children, 1);
+ EXPECT_EQ(UnpackMetadata(decoded->children[0]->metadata), field_key_values);
+
+ // ...and through the writer and the stream reader
+ nanoarrow::UniqueBuffer output;
+ nanoarrow::ipc::UniqueOutputStream output_stream;
+ ASSERT_EQ(ArrowIpcOutputStreamInitBuffer(output_stream.get(), output.get()),
+ NANOARROW_OK);
+ nanoarrow::ipc::UniqueWriter writer;
+ ASSERT_EQ(ArrowIpcWriterInit(writer.get(), output_stream.get()), NANOARROW_OK);
+ ASSERT_EQ(ArrowIpcWriterWriteSchema(writer.get(), schema.get(), &error), NANOARROW_OK)
+ << error.message;
+ ASSERT_EQ(ArrowIpcWriterWriteArrayView(writer.get(), nullptr, &error), NANOARROW_OK)
+ << error.message;
+
+ struct ArrowIpcInputStream input_stream;
+ ASSERT_EQ(ArrowIpcInputStreamInitBuffer(&input_stream, output.get()), NANOARROW_OK);
+ nanoarrow::UniqueArrayStream reader;
+ ASSERT_EQ(ArrowIpcArrayStreamReaderInit(reader.get(), &input_stream, nullptr),
+ NANOARROW_OK);
+ nanoarrow::UniqueSchema read_schema;
+ ASSERT_EQ(ArrowArrayStreamGetSchema(reader.get(), read_schema.get(), &error),
+ NANOARROW_OK)
+ << error.message;
+ EXPECT_EQ(UnpackMetadata(read_schema->metadata), schema_key_values);
+ ASSERT_EQ(read_schema->n_children, 1);
+ EXPECT_EQ(UnpackMetadata(read_schema->children[0]->metadata), field_key_values);
+}
+
TEST(NanoarrowIpcTest, NanoarrowIpcEncoderSchemaMessageMetadata) {
nanoarrow::ipc::UniqueEncoder encoder;
ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK);