You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by fo...@apache.org on 2026/10/14 14:23:50 UTC
(avro) branch main updated: AVRO-4360: [C++] Fix binary floating-point byte order (#4043)
This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/main by this push:
new 3a207fa6e8 AVRO-4360: [C++] Fix binary floating-point byte order (#4043)
3a207fa6e8 is described below
commit 3a207fa6e87c535b95eafca4cf5a6ef524d77df2
Author: Manu Zhang <Ow...@gmail.com>
AuthorDate: Wed Oct 14 22:23:42 2026 +0800
AVRO-4360: [C++] Fix binary floating-point byte order (#4043)
Use the little-endian IEEE 754 wire format required by Avro when
encoding and decoding floats and doubles on big-endian hosts.
Add independent wire-byte regressions for signed zero, finite limits,
infinities and NaN payloads across fragmented streams.
Co-authored-by: Codex <co...@openai.com>
---
lang/c++/impl/BinaryDecoder.cc | 21 +++++++++----
lang/c++/impl/BinaryEncoder.cc | 16 +++++++---
lang/c++/test/CodecTests.cc | 71 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 98 insertions(+), 10 deletions(-)
diff --git a/lang/c++/impl/BinaryDecoder.cc b/lang/c++/impl/BinaryDecoder.cc
index b334de7cf5..e6dcbe971f 100644
--- a/lang/c++/impl/BinaryDecoder.cc
+++ b/lang/c++/impl/BinaryDecoder.cc
@@ -19,6 +19,9 @@
#include "Decoder.hh"
#include "Exception.hh"
#include "Zigzag.hh"
+#include <algorithm>
+#include <array>
+#include <bit>
#include <memory>
namespace avro {
@@ -90,15 +93,21 @@ int64_t BinaryDecoder::decodeLong() {
}
float BinaryDecoder::decodeFloat() {
- float result;
- in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(float));
- return result;
+ std::array<uint8_t, 4> bytes;
+ in_.readBytes(bytes.data(), bytes.size());
+ if constexpr (std::endian::native == std::endian::big) {
+ std::reverse(bytes.begin(), bytes.end());
+ }
+ return std::bit_cast<float>(bytes);
}
double BinaryDecoder::decodeDouble() {
- double result;
- in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(double));
- return result;
+ std::array<uint8_t, 8> bytes;
+ in_.readBytes(bytes.data(), bytes.size());
+ if constexpr (std::endian::native == std::endian::big) {
+ std::reverse(bytes.begin(), bytes.end());
+ }
+ return std::bit_cast<double>(bytes);
}
size_t BinaryDecoder::doDecodeLength() {
diff --git a/lang/c++/impl/BinaryEncoder.cc b/lang/c++/impl/BinaryEncoder.cc
index fcb80bd43d..bdd1dd4470 100644
--- a/lang/c++/impl/BinaryEncoder.cc
+++ b/lang/c++/impl/BinaryEncoder.cc
@@ -18,7 +18,9 @@
#include "Encoder.hh"
#include "Zigzag.hh"
+#include <algorithm>
#include <array>
+#include <bit>
namespace avro {
@@ -79,13 +81,19 @@ void BinaryEncoder::encodeLong(int64_t l) {
}
void BinaryEncoder::encodeFloat(float f) {
- const auto *p = reinterpret_cast<const uint8_t *>(&f);
- out_.writeBytes(p, sizeof(float));
+ auto bytes = std::bit_cast<std::array<uint8_t, 4>>(f);
+ if constexpr (std::endian::native == std::endian::big) {
+ std::reverse(bytes.begin(), bytes.end());
+ }
+ out_.writeBytes(bytes.data(), bytes.size());
}
void BinaryEncoder::encodeDouble(double d) {
- const auto *p = reinterpret_cast<const uint8_t *>(&d);
- out_.writeBytes(p, sizeof(double));
+ auto bytes = std::bit_cast<std::array<uint8_t, 8>>(d);
+ if constexpr (std::endian::native == std::endian::big) {
+ std::reverse(bytes.begin(), bytes.end());
+ }
+ out_.writeBytes(bytes.data(), bytes.size());
}
void BinaryEncoder::encodeString(const std::string &s) {
diff --git a/lang/c++/test/CodecTests.cc b/lang/c++/test/CodecTests.cc
index 59aa023dbd..428f152bb8 100644
--- a/lang/c++/test/CodecTests.cc
+++ b/lang/c++/test/CodecTests.cc
@@ -25,6 +25,8 @@
#include "Specific.hh"
#include "ValidSchema.hh"
+#include <array>
+#include <bit>
#include <boost/bind/bind.hpp>
#include <cstdint>
#include <functional>
@@ -1982,6 +1984,73 @@ static void testStreamLifetimes() {
}
}
+template<typename T, typename Bits>
+static void checkFloatingPointBinaryCodec(Bits bits, const std::array<uint8_t, sizeof(Bits)> &expected) {
+ for (size_t chunkSize : {1, 3, 16}) {
+ BOOST_TEST_CONTEXT("bits=" << bits << ", chunkSize=" << chunkSize) {
+ auto output = memoryOutputStream(chunkSize);
+ auto encoder = binaryEncoder();
+ encoder->init(*output);
+ avro::encode(*encoder, std::bit_cast<T>(bits));
+ encoder->flush();
+
+ const auto actual = snapshot(*output);
+ BOOST_CHECK_EQUAL_COLLECTIONS(actual->begin(), actual->end(), expected.begin(), expected.end());
+
+ // Decode independent wire bytes, so matching encoder/decoder bugs cannot cancel out.
+ auto wire = memoryOutputStream(chunkSize);
+ StreamWriter writer;
+ writer.reset(*wire);
+ writer.writeBytes(expected.data(), expected.size());
+ writer.flush();
+ auto input = memoryInputStream(*wire);
+ auto decoder = binaryDecoder();
+ decoder->init(*input);
+ T value;
+ avro::decode(*decoder, value);
+ BOOST_CHECK_EQUAL(std::bit_cast<Bits>(value), bits);
+ }
+ }
+}
+
+static void testFloatBinaryByteOrder() {
+ const std::pair<uint32_t, std::array<uint8_t, 4>> cases[] = {
+ {0x00000000, {0x00, 0x00, 0x00, 0x00}}, // +0
+ {0x80000000, {0x00, 0x00, 0x00, 0x80}}, // -0
+ {0x3f800000, {0x00, 0x00, 0x80, 0x3f}}, // 1
+ {0xc0200000, {0x00, 0x00, 0x20, 0xc0}}, // -2.5
+ {0x00000001, {0x01, 0x00, 0x00, 0x00}}, // Smallest subnormal
+ {0x00800000, {0x00, 0x00, 0x80, 0x00}}, // Smallest positive normal
+ {0x7f7fffff, {0xff, 0xff, 0x7f, 0x7f}}, // Largest finite value
+ {0x7f800000, {0x00, 0x00, 0x80, 0x7f}}, // +infinity
+ {0xff800000, {0x00, 0x00, 0x80, 0xff}}, // -infinity
+ {0x7fc12345, {0x45, 0x23, 0xc1, 0x7f}}, // Quiet NaN with payload
+ {0xffc54321, {0x21, 0x43, 0xc5, 0xff}}, // Negative quiet NaN with payload
+ };
+ for (const auto &[bits, bytes] : cases) {
+ checkFloatingPointBinaryCodec<float>(bits, bytes);
+ }
+}
+
+static void testDoubleBinaryByteOrder() {
+ const std::pair<uint64_t, std::array<uint8_t, 8>> cases[] = {
+ {0x0000000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, // +0
+ {0x8000000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}}, // -0
+ {0x3ff0000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f}}, // 1
+ {0xc004000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xc0}}, // -2.5
+ {0x0000000000000001, {0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, // Smallest subnormal
+ {0x0010000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00}}, // Smallest positive normal
+ {0x7fefffffffffffff, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0x7f}}, // Largest finite value
+ {0x7ff0000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7f}}, // +infinity
+ {0xfff0000000000000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0xff}}, // -infinity
+ {0x7ff8123456789abc, {0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, 0xf8, 0x7f}}, // Quiet NaN with payload
+ {0xfff8abcdef123456, {0x56, 0x34, 0x12, 0xef, 0xcd, 0xab, 0xf8, 0xff}}, // Negative quiet NaN with payload
+ };
+ for (const auto &[bits, bytes] : cases) {
+ checkFloatingPointBinaryCodec<double>(bits, bytes);
+ }
+}
+
static void testLimits(const EncoderPtr &e, const DecoderPtr &d) {
std::unique_ptr<OutputStream> s1 = memoryOutputStream();
{
@@ -2154,6 +2223,8 @@ init_unit_test_suite(int, char *[]) {
auto *ts = BOOST_TEST_SUITE("Avro C++ unit tests for codecs");
avro::parsing::add_tests(*ts);
ts->add(BOOST_TEST_CASE(avro::testStreamLifetimes));
+ ts->add(BOOST_TEST_CASE(avro::testFloatBinaryByteOrder));
+ ts->add(BOOST_TEST_CASE(avro::testDoubleBinaryByteOrder));
ts->add(BOOST_TEST_CASE(avro::testLimitsBinaryCodec));
ts->add(BOOST_TEST_CASE(avro::testLimitsJsonCodec));
ts->add(BOOST_PARAM_TEST_CASE(&avro::testJson, avro::jsonData,