You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "HeartLinked (via GitHub)" <gi...@apache.org> on 2025/08/20 07:44:16 UTC
[PR] feat: implement Literal class with serialization [iceberg-cpp]
HeartLinked opened a new pull request, #185:
URL: https://github.com/apache/iceberg-cpp/pull/185
This PR implements the Literal class for the Iceberg expression system, providing comprehensive support for literal values in query expressions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2297440489
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2315374240
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Value{std::monostate{}};
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
Review Comment:
no. All these 4 types should be int64_t. `using Value = std::variant<std::monostate, bool,int32_t, int64_t, // for long, timestamp, timestamp_tz, time`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2297690740
##########
src/iceberg/expression/literal.cc:
##########
@@ -294,13 +388,21 @@ std::string Literal::ToString() const {
}
return result;
}
+ case TypeId::kDate: {
+ return FormatDate(std::get<int32_t>(value_));
+ }
+ case TypeId::kTime: {
+ return FormatTime(std::get<int64_t>(value_));
+ }
+ case TypeId::kTimestamp: {
+ return FormatTimestamp(std::get<int64_t>(value_));
+ }
+ case TypeId::kTimestampTz: {
+ return FormatTimestampTz(std::get<int64_t>(value_));
+ }
case TypeId::kDecimal:
case TypeId::kUuid:
- case TypeId::kFixed:
- case TypeId::kDate:
- case TypeId::kTime:
- case TypeId::kTimestamp:
- case TypeId::kTimestampTz: {
+ case TypeId::kFixed: {
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "zhjwpku (via GitHub)" <gi...@apache.org>.
zhjwpku commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2368102928
##########
src/iceberg/expression/literal.cc:
##########
@@ -151,11 +155,11 @@ Literal Literal::Binary(std::vector<uint8_t> value) {
Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
std::shared_ptr<PrimitiveType> type) {
- return NotImplemented("Deserialization of Literal is not implemented yet");
+ return Conversions::FromBytes(type, data);
Review Comment:
Why wasn't the suggested code adopted? I share the same concern ;)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2379344988
##########
src/iceberg/expression/literal.h:
##########
@@ -71,6 +71,7 @@ class ICEBERG_EXPORT Literal {
static Literal Double(double value);
static Literal String(std::string value);
static Literal Binary(std::vector<uint8_t> value);
+ static Literal Fixed(std::vector<uint8_t> value, int32_t length);
Review Comment:
Exactly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko merged PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "mapleFU (via GitHub)" <gi...@apache.org>.
mapleFU commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2296093444
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
Review Comment:
why this type is not passed by reference?
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if (data.size() < sizeof(T)) {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ value = std::byteswap(value);
+ }
Review Comment:
Maybe we can add a `FromLittleEndian` and `ToLittleEndian` function here
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
Review Comment:
I found a weird thing here. This part of the code only checks the buffer size in few types, it assumes the buffer-size equals to the size upper layer passed in. Should we do some check here?
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
Review Comment:
The style here mixes assign and insert back, the code is right but I think keep a same style might be better ( .i.e using `result.insert` )
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
Review Comment:
Maybe can print the literal here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2375102363
##########
src/iceberg/expression/literal.cc:
##########
@@ -249,6 +255,20 @@ std::partial_ordering Literal::operator<=>(const Literal& other) const {
return this_val <=> other_val;
}
+ case TypeId::kFixed: {
+ // Fixed types can only be compared if they have the same length
+ auto& this_fixed_type = static_cast<const FixedType&>(*type_);
+ auto& other_fixed_type = static_cast<const FixedType&>(*other.type_);
+
+ if (this_fixed_type.length() != other_fixed_type.length()) {
+ return std::partial_ordering::unordered;
+ }
Review Comment:
I think we can remove line 259 to line 256 if we modify line 197 to
```
// If types are different, comparison is unordered
if (*type_ != *other.type_) {
return std::partial_ordering::unordered;
}
```
##########
src/iceberg/expression/literal.h:
##########
@@ -71,6 +71,7 @@ class ICEBERG_EXPORT Literal {
static Literal Double(double value);
static Literal String(std::string value);
static Literal Binary(std::vector<uint8_t> value);
+ static Literal Fixed(std::vector<uint8_t> value, int32_t length);
Review Comment:
```suggestion
static Literal Fixed(std::vector<uint8_t> value);
```
I think a single vector is enough and the length can be deduced from value.size(). Otherwise, it is annoying when `length != value.size()`, should we throw then?
##########
src/iceberg/expression/literal.cc:
##########
@@ -249,6 +255,20 @@ std::partial_ordering Literal::operator<=>(const Literal& other) const {
return this_val <=> other_val;
}
+ case TypeId::kFixed: {
+ // Fixed types can only be compared if they have the same length
+ auto& this_fixed_type = static_cast<const FixedType&>(*type_);
+ auto& other_fixed_type = static_cast<const FixedType&>(*other.type_);
+
+ if (this_fixed_type.length() != other_fixed_type.length()) {
+ return std::partial_ordering::unordered;
+ }
+
Review Comment:
```suggestion
```
I'd suggest to remove these lines and change the line 196 above as below:
```cpp
// If types are different, comparison is unordered
if (*type_ != *other.type_) {
return std::partial_ordering::unordered;
}
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -149,13 +150,17 @@ Literal Literal::Binary(std::vector<uint8_t> value) {
return {Value{std::move(value)}, binary()};
}
+Literal Literal::Fixed(std::vector<uint8_t> value, int32_t length) {
+ return {Value{std::move(value)}, fixed(length)};
Review Comment:
nit: add `ICEBERG_DCHECK(value.size() == length)`
##########
test/literal_test.cc:
##########
@@ -383,4 +383,211 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
+};
+
+TEST_P(LiteralSerializationParam, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(*literal_result, param.expected_literal);
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(*final_literal, param.expected_literal);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerialization, LiteralSerializationParam,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "IntMaxValue", {255, 255, 255, 127}, Literal::Int(2147483647), int32()},
+ LiteralRoundTripParam{
+ "IntMinValue", {0, 0, 0, 128}, Literal::Int(-2147483648), int32()},
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"LongMaxValue",
+ {255, 255, 255, 255, 255, 255, 255, 127},
+ Literal::Long(std::numeric_limits<int64_t>::max()),
+ int64()},
+ LiteralRoundTripParam{"LongMinValue",
+ {0, 0, 0, 0, 0, 0, 0, 128},
+ Literal::Long(std::numeric_limits<int64_t>::min()),
+ int64()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{"FloatNegativeInfinity",
+ {0, 0, 128, 255},
+ Literal::Float(-std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatMaxValue",
+ {255, 255, 127, 127},
+ Literal::Float(std::numeric_limits<float>::max()),
+ float32()},
+ LiteralRoundTripParam{"FloatMinValue",
+ {255, 255, 127, 255},
+ Literal::Float(std::numeric_limits<float>::lowest()),
+ float32()},
+
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"DoubleNegativeInfinity",
+ {0, 0, 0, 0, 0, 0, 240, 255},
+ Literal::Double(-std::numeric_limits<double>::infinity()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMaxValue",
+ {255, 255, 255, 255, 255, 255, 239, 127},
+ Literal::Double(std::numeric_limits<double>::max()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMinValue",
+ {255, 255, 255, 255, 255, 255, 239, 255},
+ Literal::Double(std::numeric_limits<double>::lowest()),
+ float64()},
+
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{
+ "StringLong",
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ Literal::String("AAAAAAAAAAAAAAAA"),
+ string()},
+
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ LiteralRoundTripParam{"BinarySingleByte", {42}, Literal::Binary({42}), binary()},
+
+ // Fixed type
+ LiteralRoundTripParam{"FixedLength4",
+ {0x01, 0x02, 0x03, 0x04},
+ Literal::Fixed({0x01, 0x02, 0x03, 0x04}, 4),
+ fixed(4)},
+ LiteralRoundTripParam{
+ "FixedLength8",
+ {0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11},
+ Literal::Fixed({0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11}, 8),
+ fixed(8)},
+ LiteralRoundTripParam{
+ "FixedLength16",
+ {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F},
+ Literal::Fixed({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F},
+ 16),
+ fixed(16)},
+ LiteralRoundTripParam{
+ "FixedSingleByte", {0xFF}, Literal::Fixed({0xFF}, 1), fixed(1)},
+
+ // Temporal types
+ LiteralRoundTripParam{"DateEpoch", {0, 0, 0, 0}, Literal::Date(0), date()},
+ LiteralRoundTripParam{"DateNextDay", {1, 0, 0, 0}, Literal::Date(1), date()},
+ LiteralRoundTripParam{"DateY2K", {205, 42, 0, 0}, Literal::Date(10957), date()},
+ LiteralRoundTripParam{
+ "DateNegative", {255, 255, 255, 255}, Literal::Date(-1), date()},
+
+ LiteralRoundTripParam{
+ "TimeMidnight", {0, 0, 0, 0, 0, 0, 0, 0}, Literal::Time(0), time()},
+ LiteralRoundTripParam{"TimeNoon",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Time(45045123456),
+ time()},
+ LiteralRoundTripParam{
+ "TimeOneSecond", {64, 66, 15, 0, 0, 0, 0, 0}, Literal::Time(1000000), time()},
+
+ LiteralRoundTripParam{"TimestampEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::Timestamp(0),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampOneSecond",
+ {64, 66, 15, 0, 0, 0, 0, 0},
+ Literal::Timestamp(1000000),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampNoon2024",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Timestamp(45045123456),
+ timestamp()},
+
+ LiteralRoundTripParam{"TimestampTzEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::TimestampTz(0),
+ timestamp_tz()},
+ LiteralRoundTripParam{"TimestampTzOneHour",
+ {0, 164, 147, 214, 0, 0, 0, 0},
+ Literal::TimestampTz(3600000000),
+ timestamp_tz()}),
+
+ [](const testing::TestParamInfo<LiteralSerializationParam::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationTest, EmptyString) {
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ auto deserialize_result = Literal::Deserialize(*empty_bytes, string());
+ EXPECT_THAT(deserialize_result, IsError(ErrorKind::kInvalidArgument));
+}
+
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(std::get<int64_t>(long_result->value()), 32L);
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(std::get<double>(double_result->value()), 1.0);
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
Review Comment:
```suggestion
```
ditto
##########
test/literal_test.cc:
##########
@@ -383,4 +383,211 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
+};
+
+TEST_P(LiteralSerializationParam, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(*literal_result, param.expected_literal);
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(*final_literal, param.expected_literal);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerialization, LiteralSerializationParam,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "IntMaxValue", {255, 255, 255, 127}, Literal::Int(2147483647), int32()},
+ LiteralRoundTripParam{
+ "IntMinValue", {0, 0, 0, 128}, Literal::Int(-2147483648), int32()},
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"LongMaxValue",
+ {255, 255, 255, 255, 255, 255, 255, 127},
+ Literal::Long(std::numeric_limits<int64_t>::max()),
+ int64()},
+ LiteralRoundTripParam{"LongMinValue",
+ {0, 0, 0, 0, 0, 0, 0, 128},
+ Literal::Long(std::numeric_limits<int64_t>::min()),
+ int64()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{"FloatNegativeInfinity",
+ {0, 0, 128, 255},
+ Literal::Float(-std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatMaxValue",
+ {255, 255, 127, 127},
+ Literal::Float(std::numeric_limits<float>::max()),
+ float32()},
+ LiteralRoundTripParam{"FloatMinValue",
+ {255, 255, 127, 255},
+ Literal::Float(std::numeric_limits<float>::lowest()),
+ float32()},
+
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"DoubleNegativeInfinity",
+ {0, 0, 0, 0, 0, 0, 240, 255},
+ Literal::Double(-std::numeric_limits<double>::infinity()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMaxValue",
+ {255, 255, 255, 255, 255, 255, 239, 127},
+ Literal::Double(std::numeric_limits<double>::max()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMinValue",
+ {255, 255, 255, 255, 255, 255, 239, 255},
+ Literal::Double(std::numeric_limits<double>::lowest()),
+ float64()},
+
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{
+ "StringLong",
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ Literal::String("AAAAAAAAAAAAAAAA"),
+ string()},
+
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ LiteralRoundTripParam{"BinarySingleByte", {42}, Literal::Binary({42}), binary()},
+
+ // Fixed type
+ LiteralRoundTripParam{"FixedLength4",
+ {0x01, 0x02, 0x03, 0x04},
+ Literal::Fixed({0x01, 0x02, 0x03, 0x04}, 4),
+ fixed(4)},
+ LiteralRoundTripParam{
+ "FixedLength8",
+ {0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11},
+ Literal::Fixed({0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11}, 8),
+ fixed(8)},
+ LiteralRoundTripParam{
+ "FixedLength16",
+ {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F},
+ Literal::Fixed({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F},
+ 16),
+ fixed(16)},
+ LiteralRoundTripParam{
+ "FixedSingleByte", {0xFF}, Literal::Fixed({0xFF}, 1), fixed(1)},
+
+ // Temporal types
+ LiteralRoundTripParam{"DateEpoch", {0, 0, 0, 0}, Literal::Date(0), date()},
+ LiteralRoundTripParam{"DateNextDay", {1, 0, 0, 0}, Literal::Date(1), date()},
+ LiteralRoundTripParam{"DateY2K", {205, 42, 0, 0}, Literal::Date(10957), date()},
+ LiteralRoundTripParam{
+ "DateNegative", {255, 255, 255, 255}, Literal::Date(-1), date()},
+
+ LiteralRoundTripParam{
+ "TimeMidnight", {0, 0, 0, 0, 0, 0, 0, 0}, Literal::Time(0), time()},
+ LiteralRoundTripParam{"TimeNoon",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Time(45045123456),
+ time()},
+ LiteralRoundTripParam{
+ "TimeOneSecond", {64, 66, 15, 0, 0, 0, 0, 0}, Literal::Time(1000000), time()},
+
+ LiteralRoundTripParam{"TimestampEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::Timestamp(0),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampOneSecond",
+ {64, 66, 15, 0, 0, 0, 0, 0},
+ Literal::Timestamp(1000000),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampNoon2024",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Timestamp(45045123456),
+ timestamp()},
+
+ LiteralRoundTripParam{"TimestampTzEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::TimestampTz(0),
+ timestamp_tz()},
+ LiteralRoundTripParam{"TimestampTzOneHour",
+ {0, 164, 147, 214, 0, 0, 0, 0},
+ Literal::TimestampTz(3600000000),
+ timestamp_tz()}),
+
+ [](const testing::TestParamInfo<LiteralSerializationParam::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationTest, EmptyString) {
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ auto deserialize_result = Literal::Deserialize(*empty_bytes, string());
+ EXPECT_THAT(deserialize_result, IsError(ErrorKind::kInvalidArgument));
Review Comment:
Could you double check the spec and java impl on how to deal with empty values? I think this is a valid case which shouldn't return an error.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() != sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kBoolean>(const Literal::Value& value) {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kString>(const Literal::Value& value) {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kBinary>(const Literal::Value& value) {
+ return std::get<std::vector<uint8_t>>(value);
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kFixed>(const Literal::Value& value) {
+ if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type =
+ std::visit([](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
Review Comment:
```suggestion
return std::get<std::vector<uint8_t>>(value);
```
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() != sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kBoolean>(const Literal::Value& value) {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kString>(const Literal::Value& value) {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kBinary>(const Literal::Value& value) {
+ return std::get<std::vector<uint8_t>>(value);
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kFixed>(const Literal::Value& value) {
+ if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type =
+ std::visit([](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kBoolean)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kString)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kBinary)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFixed)
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Cannot deserialize empty value");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
Review Comment:
Generally I don't think it is a good practice to add blank lines between cases which makes the switch block lengthy.
##########
test/literal_test.cc:
##########
@@ -136,8 +136,8 @@ TEST(LiteralTest, LongCastTo) {
EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
}
+// Test overflow cases
Review Comment:
```suggestion
```
The comment is redundant.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() != sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kBoolean>(const Literal::Value& value) {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kString>(const Literal::Value& value) {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kBinary>(const Literal::Value& value) {
+ return std::get<std::vector<uint8_t>>(value);
+}
+
+template <>
+Result<std::vector<uint8_t>> ToBytesImpl<TypeId::kFixed>(const Literal::Value& value) {
+ if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type =
+ std::visit([](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kBoolean)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kString)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kBinary)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFixed)
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Cannot deserialize empty value");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() < 8) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ }
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() < 8) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Value{static_cast<double>(float_value)};
+ } else {
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Value{double_value};
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::Value{
+ std::string(reinterpret_cast<const char*>(data.data()), data.size())};
+ }
Review Comment:
```suggestion
case TypeId::kString:
return Literal::Value{
std::string(reinterpret_cast<const char*>(data.data()), data.size())};
```
Usually we don't add curly braces to a single-line case.
##########
test/literal_test.cc:
##########
@@ -383,4 +383,211 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
+};
+
+TEST_P(LiteralSerializationParam, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(*literal_result, param.expected_literal);
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(*final_literal, param.expected_literal);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerialization, LiteralSerializationParam,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "IntMaxValue", {255, 255, 255, 127}, Literal::Int(2147483647), int32()},
+ LiteralRoundTripParam{
+ "IntMinValue", {0, 0, 0, 128}, Literal::Int(-2147483648), int32()},
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"LongMaxValue",
+ {255, 255, 255, 255, 255, 255, 255, 127},
+ Literal::Long(std::numeric_limits<int64_t>::max()),
+ int64()},
+ LiteralRoundTripParam{"LongMinValue",
+ {0, 0, 0, 0, 0, 0, 0, 128},
+ Literal::Long(std::numeric_limits<int64_t>::min()),
+ int64()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{"FloatNegativeInfinity",
+ {0, 0, 128, 255},
+ Literal::Float(-std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatMaxValue",
+ {255, 255, 127, 127},
+ Literal::Float(std::numeric_limits<float>::max()),
+ float32()},
+ LiteralRoundTripParam{"FloatMinValue",
+ {255, 255, 127, 255},
+ Literal::Float(std::numeric_limits<float>::lowest()),
+ float32()},
+
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"DoubleNegativeInfinity",
+ {0, 0, 0, 0, 0, 0, 240, 255},
+ Literal::Double(-std::numeric_limits<double>::infinity()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMaxValue",
+ {255, 255, 255, 255, 255, 255, 239, 127},
+ Literal::Double(std::numeric_limits<double>::max()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMinValue",
+ {255, 255, 255, 255, 255, 255, 239, 255},
+ Literal::Double(std::numeric_limits<double>::lowest()),
+ float64()},
+
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{
+ "StringLong",
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ Literal::String("AAAAAAAAAAAAAAAA"),
+ string()},
+
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ LiteralRoundTripParam{"BinarySingleByte", {42}, Literal::Binary({42}), binary()},
+
+ // Fixed type
+ LiteralRoundTripParam{"FixedLength4",
+ {0x01, 0x02, 0x03, 0x04},
+ Literal::Fixed({0x01, 0x02, 0x03, 0x04}, 4),
+ fixed(4)},
+ LiteralRoundTripParam{
+ "FixedLength8",
+ {0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11},
+ Literal::Fixed({0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11}, 8),
+ fixed(8)},
+ LiteralRoundTripParam{
+ "FixedLength16",
+ {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F},
+ Literal::Fixed({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F},
+ 16),
+ fixed(16)},
+ LiteralRoundTripParam{
+ "FixedSingleByte", {0xFF}, Literal::Fixed({0xFF}, 1), fixed(1)},
+
+ // Temporal types
+ LiteralRoundTripParam{"DateEpoch", {0, 0, 0, 0}, Literal::Date(0), date()},
+ LiteralRoundTripParam{"DateNextDay", {1, 0, 0, 0}, Literal::Date(1), date()},
+ LiteralRoundTripParam{"DateY2K", {205, 42, 0, 0}, Literal::Date(10957), date()},
+ LiteralRoundTripParam{
+ "DateNegative", {255, 255, 255, 255}, Literal::Date(-1), date()},
+
+ LiteralRoundTripParam{
+ "TimeMidnight", {0, 0, 0, 0, 0, 0, 0, 0}, Literal::Time(0), time()},
+ LiteralRoundTripParam{"TimeNoon",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Time(45045123456),
+ time()},
+ LiteralRoundTripParam{
+ "TimeOneSecond", {64, 66, 15, 0, 0, 0, 0, 0}, Literal::Time(1000000), time()},
+
+ LiteralRoundTripParam{"TimestampEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::Timestamp(0),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampOneSecond",
+ {64, 66, 15, 0, 0, 0, 0, 0},
+ Literal::Timestamp(1000000),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampNoon2024",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Timestamp(45045123456),
+ timestamp()},
+
+ LiteralRoundTripParam{"TimestampTzEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::TimestampTz(0),
+ timestamp_tz()},
+ LiteralRoundTripParam{"TimestampTzOneHour",
+ {0, 164, 147, 214, 0, 0, 0, 0},
+ Literal::TimestampTz(3600000000),
+ timestamp_tz()}),
+
+ [](const testing::TestParamInfo<LiteralSerializationParam::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationTest, EmptyString) {
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ auto deserialize_result = Literal::Deserialize(*empty_bytes, string());
+ EXPECT_THAT(deserialize_result, IsError(ErrorKind::kInvalidArgument));
+}
+
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(std::get<int64_t>(long_result->value()), 32L);
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
Review Comment:
```suggestion
```
These lines are meaningless to this case.
##########
test/literal_test.cc:
##########
@@ -383,4 +383,211 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
+};
+
+TEST_P(LiteralSerializationParam, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(*literal_result, param.expected_literal);
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(*final_literal, param.expected_literal);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerialization, LiteralSerializationParam,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "IntMaxValue", {255, 255, 255, 127}, Literal::Int(2147483647), int32()},
+ LiteralRoundTripParam{
+ "IntMinValue", {0, 0, 0, 128}, Literal::Int(-2147483648), int32()},
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"LongMaxValue",
+ {255, 255, 255, 255, 255, 255, 255, 127},
+ Literal::Long(std::numeric_limits<int64_t>::max()),
+ int64()},
+ LiteralRoundTripParam{"LongMinValue",
+ {0, 0, 0, 0, 0, 0, 0, 128},
+ Literal::Long(std::numeric_limits<int64_t>::min()),
+ int64()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{"FloatNegativeInfinity",
+ {0, 0, 128, 255},
+ Literal::Float(-std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatMaxValue",
+ {255, 255, 127, 127},
+ Literal::Float(std::numeric_limits<float>::max()),
+ float32()},
+ LiteralRoundTripParam{"FloatMinValue",
+ {255, 255, 127, 255},
+ Literal::Float(std::numeric_limits<float>::lowest()),
+ float32()},
+
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"DoubleNegativeInfinity",
+ {0, 0, 0, 0, 0, 0, 240, 255},
+ Literal::Double(-std::numeric_limits<double>::infinity()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMaxValue",
+ {255, 255, 255, 255, 255, 255, 239, 127},
+ Literal::Double(std::numeric_limits<double>::max()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMinValue",
+ {255, 255, 255, 255, 255, 255, 239, 255},
+ Literal::Double(std::numeric_limits<double>::lowest()),
+ float64()},
+
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{
+ "StringLong",
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ Literal::String("AAAAAAAAAAAAAAAA"),
+ string()},
+
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ LiteralRoundTripParam{"BinarySingleByte", {42}, Literal::Binary({42}), binary()},
+
+ // Fixed type
+ LiteralRoundTripParam{"FixedLength4",
+ {0x01, 0x02, 0x03, 0x04},
+ Literal::Fixed({0x01, 0x02, 0x03, 0x04}, 4),
+ fixed(4)},
+ LiteralRoundTripParam{
+ "FixedLength8",
+ {0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11},
+ Literal::Fixed({0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11}, 8),
+ fixed(8)},
+ LiteralRoundTripParam{
+ "FixedLength16",
+ {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F},
+ Literal::Fixed({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F},
+ 16),
+ fixed(16)},
+ LiteralRoundTripParam{
+ "FixedSingleByte", {0xFF}, Literal::Fixed({0xFF}, 1), fixed(1)},
+
+ // Temporal types
+ LiteralRoundTripParam{"DateEpoch", {0, 0, 0, 0}, Literal::Date(0), date()},
+ LiteralRoundTripParam{"DateNextDay", {1, 0, 0, 0}, Literal::Date(1), date()},
+ LiteralRoundTripParam{"DateY2K", {205, 42, 0, 0}, Literal::Date(10957), date()},
+ LiteralRoundTripParam{
+ "DateNegative", {255, 255, 255, 255}, Literal::Date(-1), date()},
+
+ LiteralRoundTripParam{
+ "TimeMidnight", {0, 0, 0, 0, 0, 0, 0, 0}, Literal::Time(0), time()},
+ LiteralRoundTripParam{"TimeNoon",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Time(45045123456),
+ time()},
+ LiteralRoundTripParam{
+ "TimeOneSecond", {64, 66, 15, 0, 0, 0, 0, 0}, Literal::Time(1000000), time()},
+
+ LiteralRoundTripParam{"TimestampEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::Timestamp(0),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampOneSecond",
+ {64, 66, 15, 0, 0, 0, 0, 0},
+ Literal::Timestamp(1000000),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampNoon2024",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Timestamp(45045123456),
+ timestamp()},
+
+ LiteralRoundTripParam{"TimestampTzEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::TimestampTz(0),
+ timestamp_tz()},
+ LiteralRoundTripParam{"TimestampTzOneHour",
+ {0, 164, 147, 214, 0, 0, 0, 0},
+ Literal::TimestampTz(3600000000),
+ timestamp_tz()}),
+
+ [](const testing::TestParamInfo<LiteralSerializationParam::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationTest, EmptyString) {
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ auto deserialize_result = Literal::Deserialize(*empty_bytes, string());
+ EXPECT_THAT(deserialize_result, IsError(ErrorKind::kInvalidArgument));
+}
+
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(std::get<int64_t>(long_result->value()), 32L);
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(std::get<double>(double_result->value()), 1.0);
Review Comment:
```suggestion
EXPECT_DOUBLE_EQ(std::get<double>(double_result->value()), 1.0);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#issuecomment-3390296012
Looks good @HeartLinked, thanks for working on this, and thanks @wgtmac @zhjwpku @mapleFU and @HuaHuaY for the review 🚀
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HuaHuaY (via GitHub)" <gi...@apache.org>.
HuaHuaY commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2370860675
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
Review Comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2315236914
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
Review Comment:
> BTW, why not `std::vector<uint8_t> WriteLittleEndian(T value)`?
I think the current `void` signature treats the `std::vector<uint8_t>&` buffer as a write target (a sink), allowing data to be appended multiple times efficiently. In contrast, returning a vector would create a new std::vector object on the heap with each call, making the existing approach superior?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2343018109
##########
test/literal_test.cc:
##########
@@ -383,4 +383,118 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParamTest
+ : public ::testing::TestWithParam<LiteralRoundTripParam> {};
+
+TEST_P(LiteralSerializationParamTest, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(literal_result->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), param.expected_literal.ToString());
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(final_literal->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), param.expected_literal.ToString());
Review Comment:
```suggestion
EXPECT_EQ(*final_literal, param.expected_literal);
```
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
Review Comment:
```suggestion
if (data.size() == sizeof(T)) [[unlikely]] {
```
Shouldn't they be strictly equal?
##########
test/literal_test.cc:
##########
@@ -383,4 +383,118 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParamTest
+ : public ::testing::TestWithParam<LiteralRoundTripParam> {};
+
+TEST_P(LiteralSerializationParamTest, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(literal_result->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), param.expected_literal.ToString());
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(final_literal->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), param.expected_literal.ToString());
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerializationTests, LiteralSerializationParamTest,
Review Comment:
Can we eliminate `Tests` and `Test` from their names to be shorter?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
Review Comment:
I think we can blindly use `std::vector<uint8_t>` for fixed type.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
Review Comment:
```suggestion
```
We have checked it in ReadLittleEndian, so no need to check it again here.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
+ data.size());
+ }
+
+ return Literal::Value{value};
Review Comment:
```suggestion
int64_t value;
if (data.size() < 8) {
// Type was promoted from int to long
ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
value = static_cast<int64_t>(int_value);
} else {
ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
value = long_value;
}
return Literal::Value{value};
```
##########
test/literal_test.cc:
##########
@@ -383,4 +383,118 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParamTest
+ : public ::testing::TestWithParam<LiteralRoundTripParam> {};
+
+TEST_P(LiteralSerializationParamTest, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(literal_result->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), param.expected_literal.ToString());
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(final_literal->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), param.expected_literal.ToString());
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerializationTests, LiteralSerializationParamTest,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ // Edge cases that fit the round-trip pattern
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+ // IEEE 754 representation for NaN and Infinity (in little-endian)
+ LiteralRoundTripParam{"FloatInfinity",
+ {0, 0, 128, 127},
+ Literal::Float(std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatNaN",
+ {0, 0, 192, 127},
+ Literal::Float(std::numeric_limits<float>::quiet_NaN()),
+ float32()}
+ // TODO(Li Feiyang): Add tests for Date, Time, Timestamp, TimestampTz
+ ),
+
+ [](const testing::TestParamInfo<LiteralSerializationParamTest::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationEdgeCaseTest, EmptyStringSerialization) {
Review Comment:
```suggestion
TEST(LiteralSerializationTest, EmptyString) {
```
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <span>
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+/// \file iceberg/util/conversions.h
+/// \brief Conversion utilities for primitive types
+
+namespace iceberg {
+
+/// \brief Conversion utilities for primitive types
+class ICEBERG_EXPORT Conversions {
+ public:
+ /// \brief Convert a literal value to bytes
+ static Result<std::vector<uint8_t>> ToBytes(const PrimitiveType& type,
+ const Literal::Value& value);
+
+ static Result<std::vector<uint8_t>> ToBytes(const Literal& literal);
Review Comment:
Please add docstring.
##########
test/literal_test.cc:
##########
@@ -383,4 +383,118 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
Review Comment:
I think it is not a good practice to have an indirection like this.
```suggestion
EXPECT_EQ(std::get<int64_t>(long_result->value()), 32L);
```
##########
test/literal_test.cc:
##########
@@ -383,4 +383,118 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
Review Comment:
This case is better to be placed after the roundtrip test.
##########
src/iceberg/util/endian.h:
##########
@@ -19,6 +19,8 @@
#pragma once
+#include <algorithm>
+#include <array>
Review Comment:
```suggestion
```
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
Review Comment:
Can we add a template specialization of ToBytesImpl for TypeId::kBoolean? Then you can simply write `DISPATCH_LITERAL_TO_BYTES(TypeId::kBoolean)`
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <span>
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+/// \file iceberg/util/conversions.h
+/// \brief Conversion utilities for primitive types
+
+namespace iceberg {
+
+/// \brief Conversion utilities for primitive types
+class ICEBERG_EXPORT Conversions {
+ public:
+ /// \brief Convert a literal value to bytes
+ static Result<std::vector<uint8_t>> ToBytes(const PrimitiveType& type,
+ const Literal::Value& value);
+
+ static Result<std::vector<uint8_t>> ToBytes(const Literal& literal);
+
+ /// \brief Convert bytes to a literal value
+ static Result<Literal::Value> FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data);
+
+ static Result<Literal> FromBytes(std::shared_ptr<PrimitiveType> type,
Review Comment:
ditto
##########
test/literal_test.cc:
##########
@@ -383,4 +383,118 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParamTest
+ : public ::testing::TestWithParam<LiteralRoundTripParam> {};
+
+TEST_P(LiteralSerializationParamTest, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(literal_result->type()->type_id(), param.expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), param.expected_literal.ToString());
Review Comment:
```suggestion
EXPECT_EQ(*literal_result, param.expected_literal);
```
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
+ data.size());
+ }
+
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ if (data.size() != sizeof(float)) {
+ return InvalidArgument("Float requires {} bytes, got {}", sizeof(float),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Value{double_value};
+ } else if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Value{static_cast<double>(float_value)};
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::Value{
+ std::string(reinterpret_cast<const char*>(data.data()), data.size())};
+ }
+
+ case TypeId::kBinary: {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
+ }
+
+ case TypeId::kFixed: {
+ if (data.size() == 16) {
+ std::array<uint8_t, 16> fixed_bytes;
+ std::ranges::copy(data, fixed_bytes.begin());
+ return Literal::Value{fixed_bytes};
+ } else {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Deserialization for type {} is not supported",
+ type.ToString());
+ }
+}
+
+Result<Literal> Conversions::FromBytes(std::shared_ptr<PrimitiveType> type,
+ std::span<const uint8_t> data) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ ICEBERG_ASSIGN_OR_RAISE(auto value, FromBytes(*type, data));
+
+ // If we got a null value (monostate), create a null Literal
+ if (std::holds_alternative<std::monostate>(value)) {
Review Comment:
Will it ever happen?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
Review Comment:
ditto
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
Review Comment:
```suggestion
return InvalidArgument("Cannot deserialize empty value");
```
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
Review Comment:
I believe we can also specialize all types below.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
+ data.size());
+ }
+
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ if (data.size() != sizeof(float)) {
+ return InvalidArgument("Float requires {} bytes, got {}", sizeof(float),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Value{double_value};
+ } else if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Value{static_cast<double>(float_value)};
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::Value{
+ std::string(reinterpret_cast<const char*>(data.data()), data.size())};
+ }
+
+ case TypeId::kBinary: {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
+ }
+
+ case TypeId::kFixed: {
+ if (data.size() == 16) {
+ std::array<uint8_t, 16> fixed_bytes;
+ std::ranges::copy(data, fixed_bytes.begin());
+ return Literal::Value{fixed_bytes};
+ } else {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
Review Comment:
Please check data.size() against type.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
+ data.size());
+ }
+
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ if (data.size() != sizeof(float)) {
+ return InvalidArgument("Float requires {} bytes, got {}", sizeof(float),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Value{double_value};
+ } else if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Value{static_cast<double>(float_value)};
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::Value{
+ std::string(reinterpret_cast<const char*>(data.data()), data.size())};
+ }
+
+ case TypeId::kBinary: {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
+ }
+
+ case TypeId::kFixed: {
+ if (data.size() == 16) {
+ std::array<uint8_t, 16> fixed_bytes;
Review Comment:
Please do not use std::array<uint8_t, 16> for fixed type, this makes cases complex.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
+ data.size());
+ }
+
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ if (data.size() != sizeof(float)) {
+ return InvalidArgument("Float requires {} bytes, got {}", sizeof(float),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Value{double_value};
+ } else if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Value{static_cast<double>(float_value)};
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
Review Comment:
```suggestion
if (data.size() < 8) {
// Type was promoted from float to double
ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
return Literal::Value{static_cast<double>(float_value)};
} else {
ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
return Literal::Value{double_value};
}
```
Let's simplify this
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format and return as vector.
+template <EndianConvertible T>
+std::vector<uint8_t> WriteLittleEndian(T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ std::vector<uint8_t> result;
+ result.insert(result.end(), bytes, bytes + sizeof(T));
+ return result;
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+template <TypeId type_id>
+Result<std::vector<uint8_t>> ToBytesImpl(const Literal::Value& value) {
+ using CppType = typename LiteralTraits<type_id>::ValueType;
+ return WriteLittleEndian(std::get<CppType>(value));
+}
+
+#define DISPATCH_LITERAL_TO_BYTES(type_id) \
+ case type_id: \
+ return ToBytesImpl<type_id>(value);
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDate)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTime)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestamp)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kTimestampTz)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
+ DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
+ case TypeId::kBoolean: {
+ return std::vector<uint8_t>{std::get<bool>(value) ? static_cast<uint8_t>(0x01)
+ : static_cast<uint8_t>(0x00)};
+ }
+
+ case TypeId::kString: {
+ const auto& str = std::get<std::string>(value);
+ return std::vector<uint8_t>(str.begin(), str.end());
+ }
+
+ case TypeId::kBinary: {
+ return std::get<std::vector<uint8_t>>(value);
+ }
+
+ case TypeId::kFixed: {
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ return std::vector<uint8_t>(fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ return std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+#undef DISPATCH_LITERAL_TO_BYTES
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ if (data.empty()) {
+ return InvalidArgument("Data cannot be empty");
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
+ data.size());
+ }
+
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ if (data.size() != sizeof(float)) {
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2379348253
##########
src/iceberg/expression/literal.h:
##########
@@ -143,11 +144,76 @@ class ICEBERG_EXPORT Literal {
private:
Literal(Value value, std::shared_ptr<PrimitiveType> type);
+ friend class Conversions;
friend class LiteralCaster;
- private:
Review Comment:
Why this gets removed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2298398395
##########
src/iceberg/util/endian.h:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <cstring>
+#include <span>
+#include <vector>
+
+#include "iceberg/result.h"
+
+/// \file iceberg/util/endian.h
+/// \brief Endianness conversion utilities
+
+namespace iceberg::util {
+
+template <typename T>
+concept LittleEndianWritable =
+ std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> ||
+ std::is_same_v<T, float> || std::is_same_v<T, double> || std::is_same_v<T, uint8_t>;
Review Comment:
```suggestion
concept LittleEndianWritable = std::is_arithmetic_v<T>;
```
Is this enough?
##########
src/iceberg/util/endian.h:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <cstring>
+#include <span>
+#include <vector>
+
+#include "iceberg/result.h"
+
+/// \file iceberg/util/endian.h
+/// \brief Endianness conversion utilities
+
+namespace iceberg::util {
+
+template <typename T>
+concept LittleEndianWritable =
+ std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> ||
+ std::is_same_v<T, float> || std::is_same_v<T, double> || std::is_same_v<T, uint8_t>;
+
+/// \brief Convert a value to little-endian format.
+template <LittleEndianWritable T>
+T ToLittleEndian(T value) {
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ return std::byteswap(value);
+ }
+ return value;
+}
+
+/// \brief Convert a value from little-endian format.
+template <LittleEndianWritable T>
+T FromLittleEndian(T value) {
+ return ToLittleEndian(value);
Review Comment:
This looks weird. I'd rather copy the above function.
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/iceberg_export.h"
+#include "iceberg/result.h"
+#include "iceberg/type.h"
Review Comment:
```suggestion
#include "iceberg/type_fwd.h"
```
##########
src/iceberg/expression/literal.h:
##########
@@ -144,8 +144,9 @@ class ICEBERG_EXPORT Literal {
Literal(Value value, std::shared_ptr<PrimitiveType> type);
friend class LiteralCaster;
+ friend class LiteralSerializer;
Review Comment:
```suggestion
```
##########
src/iceberg/util/endian.h:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <cstring>
+#include <span>
+#include <vector>
+
+#include "iceberg/result.h"
+
+/// \file iceberg/util/endian.h
+/// \brief Endianness conversion utilities
+
+namespace iceberg::util {
Review Comment:
```suggestion
namespace iceberg {
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -151,11 +155,11 @@ Literal Literal::Binary(std::vector<uint8_t> value) {
Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
std::shared_ptr<PrimitiveType> type) {
- return NotImplemented("Deserialization of Literal is not implemented yet");
+ return Conversions::FromBytes(type, data);
Review Comment:
```suggestion
return Conversions::FromBytes(std::move(type), data);
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -21,8 +21,12 @@
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <utility>
Review Comment:
```suggestion
```
##########
src/iceberg/util/endian.h:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <cstring>
+#include <span>
+#include <vector>
+
+#include "iceberg/result.h"
+
+/// \file iceberg/util/endian.h
+/// \brief Endianness conversion utilities
+
+namespace iceberg::util {
+
+template <typename T>
+concept LittleEndianWritable =
+ std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> ||
+ std::is_same_v<T, float> || std::is_same_v<T, double> || std::is_same_v<T, uint8_t>;
+
+/// \brief Convert a value to little-endian format.
+template <LittleEndianWritable T>
+T ToLittleEndian(T value) {
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ return std::byteswap(value);
+ }
+ return value;
+}
+
+/// \brief Convert a value from little-endian format.
+template <LittleEndianWritable T>
+T FromLittleEndian(T value) {
+ return ToLittleEndian(value);
+}
+
+/// \brief Write a value in little-endian format to the buffer.
+template <LittleEndianWritable T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ T le_value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
Review Comment:
```suggestion
value = ToLittleEndian(value);
const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
```
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/iceberg_export.h"
Review Comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2312908195
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <span>
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
Review Comment:
```suggestion
namespace iceberg {
```
##########
src/iceberg/CMakeLists.txt:
##########
@@ -50,7 +50,8 @@ set(ICEBERG_SOURCES
arrow_c_data_guard_internal.cc
util/murmurhash3_internal.cc
util/timepoint.cc
- util/gzip_internal.cc)
+ util/gzip_internal.cc
+ util/conversions.cc)
Review Comment:
Please sort them alphabetically.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Value{std::monostate{}};
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ auto type_name_view = ToString(type_id);
+ std::string type_name{type_name_view};
+ if (!type_name.empty()) {
+ type_name[0] = static_cast<char>(std::toupper(type_name[0]));
+ }
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
+ data.size());
+ }
+
+ return Literal::Value{value};
+ }
+
+ case TypeId::kFloat: {
+ if (data.size() != sizeof(float)) {
+ return InvalidArgument("Float requires {} bytes, got {}", sizeof(float),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 8) {
+ // Standard 8-byte double
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Value{double_value};
+ } else if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Value{static_cast<double>(float_value)};
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::Value{
+ std::string(reinterpret_cast<const char*>(data.data()), data.size())};
+ }
+
+ case TypeId::kBinary: {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
+ }
+
+ case TypeId::kFixed: {
+ if (data.size() == 16) {
+ std::array<uint8_t, 16> fixed_bytes;
+ std::ranges::copy(data, fixed_bytes.begin());
+ return Literal::Value{fixed_bytes};
+ } else {
+ return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
+ }
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Deserialization for type {} is not supported",
+ type.ToString());
+ }
+}
+
+Result<Literal> Conversions::FromBytes(std::shared_ptr<PrimitiveType> type,
+ std::span<const uint8_t> data) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ ICEBERG_ASSIGN_OR_RAISE(auto value, FromBytes(*type, data));
+
+ // If we got a null value (monostate), create a null Literal
+ if (std::holds_alternative<std::monostate>(value)) {
+ return Literal::Null(type);
+ }
+
+ return Literal(value, type);
Review Comment:
Use `std::move`?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
Review Comment:
Are you sure about this? Should we return error instead?
##########
src/iceberg/expression/literal.cc:
##########
@@ -151,11 +152,11 @@ Literal Literal::Binary(std::vector<uint8_t> value) {
Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
std::shared_ptr<PrimitiveType> type) {
- return NotImplemented("Deserialization of Literal is not implemented yet");
+ return Conversions::FromBytes(std::move(type), data);
Review Comment:
Do we actually need to add `std::move` on `std::span`?
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <span>
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+class ICEBERG_EXPORT Conversions {
+ public:
+ static Result<std::vector<uint8_t>> ToBytes(const PrimitiveType& type,
+ const Literal::Value& value);
+
+ static Result<std::vector<uint8_t>> ToBytes(const Literal& literal);
+
+ static Result<Literal::Value> FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data);
+
+ static Result<Literal> FromBytes(std::shared_ptr<PrimitiveType> type,
+ std::span<const uint8_t> data);
+};
Review Comment:
```suggestion
};
```
##########
src/iceberg/util/conversions.h:
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <span>
+#include <vector>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+class ICEBERG_EXPORT Conversions {
Review Comment:
Please add comment to public classes and functions.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
Review Comment:
```suggestion
void WriteLittleEndian(T value, std::vector<uint8_t>* buffer) {
```
Usually the output parameter is placed at last and use pointer instead of reference type.
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Value{std::monostate{}};
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
Review Comment:
Can we use template FromBytes like below to make the switch case more compact?
```
template <TypeId type_id>
Result<Literal::Value> FromBytes(std::span<const uint8_t> data)
```
##########
test/literal_test.cc:
##########
@@ -383,4 +383,142 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+void CheckBinaryRoundTrip(const std::vector<uint8_t>& input_bytes,
+ const Literal& expected_literal,
+ std::shared_ptr<PrimitiveType> type) {
+ // Deserialize from bytes
+ auto literal_result = Literal::Deserialize(input_bytes, type);
+ ASSERT_TRUE(literal_result.has_value());
+
+ // Check type and value are correct
+ EXPECT_EQ(literal_result->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), expected_literal.ToString());
+
+ // Serialize back to bytes
+ auto bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value());
+ EXPECT_EQ(*bytes_result, input_bytes);
+
+ // Deserialize again to verify
+ auto final_literal = Literal::Deserialize(*bytes_result, type);
+ ASSERT_TRUE(final_literal.has_value());
+ EXPECT_EQ(final_literal->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), expected_literal.ToString());
+}
+
+// binary serialization tests
+TEST(LiteralSerializationTest, BinaryBoolean) {
Review Comment:
Can you please use parameterized test pattern to reuse a single test case for different types?
##########
test/manifest_list_reader_test.cc:
##########
@@ -20,9 +20,10 @@
#include <arrow/filesystem/localfs.h>
#include <avro/GenericDatum.hh>
#include <gtest/gtest.h>
+#include <iceberg/avro/avro_register.h>
Review Comment:
```suggestion
```
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Value{std::monostate{}};
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
Review Comment:
Isn't this branch applied to `case TypeId::kLong` only?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
Review Comment:
Is it too verbose to add these comments?
##########
src/iceberg/expression/literal.h:
##########
@@ -144,8 +144,8 @@ class ICEBERG_EXPORT Literal {
Literal(Value value, std::shared_ptr<PrimitiveType> type);
friend class LiteralCaster;
+ friend class Conversions;
Review Comment:
Sort them alphabetically?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Value{std::monostate{}};
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else {
+ auto type_name_view = ToString(type_id);
+ std::string type_name{type_name_view};
+ if (!type_name.empty()) {
+ type_name[0] = static_cast<char>(std::toupper(type_name[0]));
+ }
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
Review Comment:
Shouldn't we use `ToString(type_id)` in the error message directly?
##########
test/literal_test.cc:
##########
@@ -383,4 +383,142 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+void CheckBinaryRoundTrip(const std::vector<uint8_t>& input_bytes,
+ const Literal& expected_literal,
+ std::shared_ptr<PrimitiveType> type) {
+ // Deserialize from bytes
+ auto literal_result = Literal::Deserialize(input_bytes, type);
+ ASSERT_TRUE(literal_result.has_value());
+
+ // Check type and value are correct
+ EXPECT_EQ(literal_result->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), expected_literal.ToString());
+
+ // Serialize back to bytes
+ auto bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value());
+ EXPECT_EQ(*bytes_result, input_bytes);
+
+ // Deserialize again to verify
+ auto final_literal = Literal::Deserialize(*bytes_result, type);
+ ASSERT_TRUE(final_literal.has_value());
+ EXPECT_EQ(final_literal->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), expected_literal.ToString());
+}
+
+// binary serialization tests
+TEST(LiteralSerializationTest, BinaryBoolean) {
+ CheckBinaryRoundTrip({1}, Literal::Boolean(true), boolean());
+ CheckBinaryRoundTrip({0}, Literal::Boolean(false), boolean());
+}
+
+TEST(LiteralSerializationTest, BinaryInt) {
+ CheckBinaryRoundTrip({32, 0, 0, 0}, Literal::Int(32), int32());
+}
+
+TEST(LiteralSerializationTest, BinaryLong) {
+ CheckBinaryRoundTrip({32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64());
+}
+
+TEST(LiteralSerializationTest, BinaryFloat) {
+ CheckBinaryRoundTrip({0, 0, 128, 63}, Literal::Float(1.0f), float32());
+}
+
+TEST(LiteralSerializationTest, BinaryDouble) {
+ CheckBinaryRoundTrip({0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64());
+}
+
+TEST(LiteralSerializationTest, BinaryString) {
+ CheckBinaryRoundTrip({105, 99, 101, 98, 101, 114, 103}, Literal::String("iceberg"),
+ string());
+}
+
+TEST(LiteralSerializationTest, BinaryData) {
+ std::vector<uint8_t> data = {0x01, 0x02, 0x03, 0xFF};
+ CheckBinaryRoundTrip(data, Literal::Binary(data), binary());
+}
+
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+// Edge case serialization tests
+TEST(LiteralSerializationTest, EdgeCases) {
+ // Negative integers
+ CheckBinaryRoundTrip({224, 255, 255, 255}, Literal::Int(-32), int32());
+ CheckBinaryRoundTrip({224, 255, 255, 255, 255, 255, 255, 255}, Literal::Long(-32),
+ int64());
+
+ // Empty string special handling: empty string -> empty bytes -> null conversion
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ // Empty bytes deserialize to null, not empty string
+ auto null_result = Literal::Deserialize(*empty_bytes, string());
+ ASSERT_TRUE(null_result.has_value());
+ EXPECT_TRUE(null_result->IsNull());
+
+ // Special floating point value serialization
+ auto nan_float = Literal::Float(std::numeric_limits<float>::quiet_NaN());
+ auto nan_bytes = nan_float.Serialize();
+ ASSERT_TRUE(nan_bytes.has_value());
+ EXPECT_EQ(nan_bytes->size(), 4);
+
+ auto inf_float = Literal::Float(std::numeric_limits<float>::infinity());
+ auto inf_bytes = inf_float.Serialize();
+ ASSERT_TRUE(inf_bytes.has_value());
+ EXPECT_EQ(inf_bytes->size(), 4);
+}
+
+// Error case serialization tests
+TEST(LiteralSerializationTest, ErrorCases) {
+ // AboveMax/BelowMin values cannot be serialized
+ auto long_literal =
+ Literal::Long(static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1);
+ auto above_max_result = long_literal.CastTo(int32());
+ ASSERT_TRUE(above_max_result.has_value());
+ EXPECT_TRUE(above_max_result->IsAboveMax());
Review Comment:
Can we directly create AboveMax?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
Review Comment:
BTW, why not `std::vector<uint8_t> WriteLittleEndian(T value)`?
##########
test/literal_test.cc:
##########
@@ -383,4 +383,142 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+void CheckBinaryRoundTrip(const std::vector<uint8_t>& input_bytes,
+ const Literal& expected_literal,
+ std::shared_ptr<PrimitiveType> type) {
+ // Deserialize from bytes
+ auto literal_result = Literal::Deserialize(input_bytes, type);
+ ASSERT_TRUE(literal_result.has_value());
+
+ // Check type and value are correct
+ EXPECT_EQ(literal_result->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), expected_literal.ToString());
+
+ // Serialize back to bytes
+ auto bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value());
+ EXPECT_EQ(*bytes_result, input_bytes);
+
+ // Deserialize again to verify
+ auto final_literal = Literal::Deserialize(*bytes_result, type);
+ ASSERT_TRUE(final_literal.has_value());
+ EXPECT_EQ(final_literal->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), expected_literal.ToString());
+}
+
+// binary serialization tests
+TEST(LiteralSerializationTest, BinaryBoolean) {
+ CheckBinaryRoundTrip({1}, Literal::Boolean(true), boolean());
+ CheckBinaryRoundTrip({0}, Literal::Boolean(false), boolean());
+}
+
+TEST(LiteralSerializationTest, BinaryInt) {
+ CheckBinaryRoundTrip({32, 0, 0, 0}, Literal::Int(32), int32());
+}
+
+TEST(LiteralSerializationTest, BinaryLong) {
+ CheckBinaryRoundTrip({32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64());
+}
+
+TEST(LiteralSerializationTest, BinaryFloat) {
+ CheckBinaryRoundTrip({0, 0, 128, 63}, Literal::Float(1.0f), float32());
+}
+
+TEST(LiteralSerializationTest, BinaryDouble) {
+ CheckBinaryRoundTrip({0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64());
+}
+
+TEST(LiteralSerializationTest, BinaryString) {
+ CheckBinaryRoundTrip({105, 99, 101, 98, 101, 114, 103}, Literal::String("iceberg"),
+ string());
+}
+
+TEST(LiteralSerializationTest, BinaryData) {
+ std::vector<uint8_t> data = {0x01, 0x02, 0x03, 0xFF};
+ CheckBinaryRoundTrip(data, Literal::Binary(data), binary());
+}
+
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+// Edge case serialization tests
+TEST(LiteralSerializationTest, EdgeCases) {
Review Comment:
This can also use parameterized test, right?
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
Review Comment:
Can we simply do the following?
```cpp
// In literal.h or a new literal_traits.h
template <TypeId type_id>
struct LiteralTraits {
using ValueType = void;
};
template <>
struct LiteralTraits<TypeId::kBoolean> {
using ValueType = bool;
};
template <>
struct LiteralTraits<TypeId::kInt> {
using ValueType = int32_t;
};
template <>
struct LiteralTraits<TypeId::kLong> {
using ValueType = int64_t;
};
// ...
// In this file
template <TypeId type_id>
Result<std::vector<uint8_t>> ToBytes(const Literal::Value& value) {
return WriteLittleEndian(std::get<LiteralTraits<type_id>>(value));
}
#define DISPATCH_LITERAL_TO_BYTES(type_id) \
case type_id: \
return ToBytes<type_id>(value);
Result<std::vector<uint8_t>> ToBytes(TypeId type_id, const Literal::Value& value) {
switch (type_id) {
DISPATCH_LITERAL_TO_BYTES(TypeId::kBoolean)
DISPATCH_LITERAL_TO_BYTES(TypeId::kInt)
DISPATCH_LITERAL_TO_BYTES(TypeId::kLong)
DISPATCH_LITERAL_TO_BYTES(TypeId::kFloat)
DISPATCH_LITERAL_TO_BYTES(TypeId::kDouble)
DISPATCH_LITERAL_TO_BYTES(TypeId::kString)
DISPATCH_LITERAL_TO_BYTES(TypeId::kBinary)
// ...
}
}
#undef DISPATCH_LITERAL_TO_BYTES
```
##########
test/literal_test.cc:
##########
@@ -383,4 +383,142 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+void CheckBinaryRoundTrip(const std::vector<uint8_t>& input_bytes,
+ const Literal& expected_literal,
+ std::shared_ptr<PrimitiveType> type) {
+ // Deserialize from bytes
+ auto literal_result = Literal::Deserialize(input_bytes, type);
+ ASSERT_TRUE(literal_result.has_value());
+
+ // Check type and value are correct
+ EXPECT_EQ(literal_result->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(literal_result->ToString(), expected_literal.ToString());
+
+ // Serialize back to bytes
+ auto bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value());
+ EXPECT_EQ(*bytes_result, input_bytes);
+
+ // Deserialize again to verify
+ auto final_literal = Literal::Deserialize(*bytes_result, type);
+ ASSERT_TRUE(final_literal.has_value());
+ EXPECT_EQ(final_literal->type()->type_id(), expected_literal.type()->type_id());
+ EXPECT_EQ(final_literal->ToString(), expected_literal.ToString());
+}
+
+// binary serialization tests
+TEST(LiteralSerializationTest, BinaryBoolean) {
+ CheckBinaryRoundTrip({1}, Literal::Boolean(true), boolean());
+ CheckBinaryRoundTrip({0}, Literal::Boolean(false), boolean());
+}
+
+TEST(LiteralSerializationTest, BinaryInt) {
+ CheckBinaryRoundTrip({32, 0, 0, 0}, Literal::Int(32), int32());
+}
+
+TEST(LiteralSerializationTest, BinaryLong) {
+ CheckBinaryRoundTrip({32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64());
+}
+
+TEST(LiteralSerializationTest, BinaryFloat) {
+ CheckBinaryRoundTrip({0, 0, 128, 63}, Literal::Float(1.0f), float32());
+}
+
+TEST(LiteralSerializationTest, BinaryDouble) {
+ CheckBinaryRoundTrip({0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64());
+}
+
+TEST(LiteralSerializationTest, BinaryString) {
+ CheckBinaryRoundTrip({105, 99, 101, 98, 101, 114, 103}, Literal::String("iceberg"),
+ string());
+}
+
+TEST(LiteralSerializationTest, BinaryData) {
+ std::vector<uint8_t> data = {0x01, 0x02, 0x03, 0xFF};
+ CheckBinaryRoundTrip(data, Literal::Binary(data), binary());
+}
+
+// Type promotion tests
+TEST(LiteralSerializationTest, TypePromotion) {
+ // 4-byte int data can be deserialized as long
+ std::vector<uint8_t> int_data = {32, 0, 0, 0};
+ auto long_result = Literal::Deserialize(int_data, int64());
+ ASSERT_TRUE(long_result.has_value());
+ EXPECT_EQ(long_result->type()->type_id(), TypeId::kLong);
+ EXPECT_EQ(long_result->ToString(), "32");
+
+ auto long_bytes = long_result->Serialize();
+ ASSERT_TRUE(long_bytes.has_value());
+ EXPECT_EQ(long_bytes->size(), 8);
+
+ // 4-byte float data can be deserialized as double
+ std::vector<uint8_t> float_data = {0, 0, 128, 63};
+ auto double_result = Literal::Deserialize(float_data, float64());
+ ASSERT_TRUE(double_result.has_value());
+ EXPECT_EQ(double_result->type()->type_id(), TypeId::kDouble);
+ EXPECT_EQ(double_result->ToString(), "1.000000");
+
+ auto double_bytes = double_result->Serialize();
+ ASSERT_TRUE(double_bytes.has_value());
+ EXPECT_EQ(double_bytes->size(), 8);
+}
+
+// Edge case serialization tests
+TEST(LiteralSerializationTest, EdgeCases) {
+ // Negative integers
+ CheckBinaryRoundTrip({224, 255, 255, 255}, Literal::Int(-32), int32());
+ CheckBinaryRoundTrip({224, 255, 255, 255, 255, 255, 255, 255}, Literal::Long(-32),
+ int64());
+
+ // Empty string special handling: empty string -> empty bytes -> null conversion
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ // Empty bytes deserialize to null, not empty string
+ auto null_result = Literal::Deserialize(*empty_bytes, string());
+ ASSERT_TRUE(null_result.has_value());
+ EXPECT_TRUE(null_result->IsNull());
+
+ // Special floating point value serialization
+ auto nan_float = Literal::Float(std::numeric_limits<float>::quiet_NaN());
+ auto nan_bytes = nan_float.Serialize();
+ ASSERT_TRUE(nan_bytes.has_value());
+ EXPECT_EQ(nan_bytes->size(), 4);
+
+ auto inf_float = Literal::Float(std::numeric_limits<float>::infinity());
+ auto inf_bytes = inf_float.Serialize();
+ ASSERT_TRUE(inf_bytes.has_value());
+ EXPECT_EQ(inf_bytes->size(), 4);
+}
+
+// Error case serialization tests
+TEST(LiteralSerializationTest, ErrorCases) {
+ // AboveMax/BelowMin values cannot be serialized
+ auto long_literal =
+ Literal::Long(static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1);
+ auto above_max_result = long_literal.CastTo(int32());
+ ASSERT_TRUE(above_max_result.has_value());
+ EXPECT_TRUE(above_max_result->IsAboveMax());
+
+ auto serialize_result = above_max_result->Serialize();
+ EXPECT_FALSE(serialize_result.has_value());
Review Comment:
Use ASSERT_THAT to verify the error message. Same apply for below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2299673313
##########
src/iceberg/util/literal_format.cc:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/literal_format.h"
+
+#include <chrono>
+#include <cstring>
+#include <iomanip>
+
+namespace iceberg {
+
+std::string FormatDate(int32_t days_since_epoch) {
+ // Convert days since Unix epoch to date
+ auto time_point =
+ std::chrono::system_clock::time_point{} + std::chrono::days{days_since_epoch};
+ auto date = std::chrono::floor<std::chrono::days>(time_point);
+ auto ymd = std::chrono::year_month_day{date};
+
+ std::ostringstream oss;
+ oss << static_cast<int>(ymd.year()) << "-" << std::setfill('0') << std::setw(2)
+ << static_cast<unsigned>(ymd.month()) << "-" << std::setfill('0') << std::setw(2)
+ << static_cast<unsigned>(ymd.day());
+ return oss.str();
+}
+
+std::string FormatTime(int64_t microseconds_since_midnight) {
+ auto hours = microseconds_since_midnight / (1000000LL * 3600);
+ auto minutes = (microseconds_since_midnight % (1000000LL * 3600)) / (1000000LL * 60);
+ auto seconds = (microseconds_since_midnight % (1000000LL * 60)) / 1000000LL;
+ auto micros = microseconds_since_midnight % 1000000LL;
+
+ std::ostringstream oss;
+ oss << std::setfill('0') << std::setw(2) << hours << ":" << std::setfill('0')
+ << std::setw(2) << minutes << ":" << std::setfill('0') << std::setw(2) << seconds
+ << "." << std::setfill('0') << std::setw(6) << micros;
+ return oss.str();
+}
+
+std::string FormatTimestamp(int64_t microseconds_since_epoch) {
Review Comment:
Please note that `timestamp` and `timestamptz` are different. `timestamp` should print the text in local timezone while `timestamptz` should use UTC.
##########
src/iceberg/util/endian.h:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <cstring>
+#include <span>
+#include <vector>
+
+#include "iceberg/result.h"
+
+/// \file iceberg/util/endian.h
+/// \brief Endianness conversion utilities
+
+namespace iceberg::util {
Review Comment:
This isn't resolved, right?
##########
src/iceberg/util/literal_format.cc:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/literal_format.h"
Review Comment:
It would be better if we can split it (i.e. adding `date_time_util.h/.cc` and do the suggested refactoring from transfrom_function.cc) from this PR and check in that first.
##########
src/iceberg/util/literal_format.cc:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/literal_format.h"
Review Comment:
```suggestion
#include "iceberg/util/date_time_util.h"
```
I think we should eventually add all functions from https://github.com/apache/iceberg/blob/main/api/src/main/java/org/apache/iceberg/util/DateTimeUtil.java. So let's rename it first. We need also move all time transformation to this utility:
https://github.com/apache/iceberg-cpp/blob/main/src/iceberg/transform_function.cc#L204-L405
##########
src/iceberg/util/literal_format.cc:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/literal_format.h"
+
+#include <chrono>
+#include <cstring>
+#include <iomanip>
+
+namespace iceberg {
+
+std::string FormatDate(int32_t days_since_epoch) {
Review Comment:
Could you please leverage std::formatter https://en.cppreference.com/w/cpp/chrono/hh_mm_ss/formatter.html instead of manual conversions? There are a bunch of specialization for `std::formatter<std::chrono::xyz>`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2297595858
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Boolean(value != 0x00);
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Int(value);
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Date(value);
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+
+ if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else {
+ const char* type_name = [type_id]() {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2378654497
##########
src/iceberg/expression/literal.h:
##########
@@ -71,6 +71,7 @@ class ICEBERG_EXPORT Literal {
static Literal Double(double value);
static Literal String(std::string value);
static Literal Binary(std::vector<uint8_t> value);
+ static Literal Fixed(std::vector<uint8_t> value, int32_t length);
Review Comment:
This comment seems contradictions with the above？So should we use
```suggestion
Literal Literal::Fixed(std::vector<uint8_t> value) {
auto length = static_cast<int32_t>(value.size());
return {Value{std::move(value)}, fixed(length)};
}
``` to implement it, not use ICEBERG_DCHECK?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "zhjwpku (via GitHub)" <gi...@apache.org>.
zhjwpku commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2368102928
##########
src/iceberg/expression/literal.cc:
##########
@@ -151,11 +155,11 @@ Literal Literal::Binary(std::vector<uint8_t> value) {
Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
std::shared_ptr<PrimitiveType> type) {
- return NotImplemented("Deserialization of Literal is not implemented yet");
+ return Conversions::FromBytes(type, data);
Review Comment:
Why wasn't the suggested code adopted? I share the same idea ;)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2370834835
##########
src/iceberg/expression/literal.cc:
##########
@@ -151,11 +155,11 @@ Literal Literal::Binary(std::vector<uint8_t> value) {
Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
std::shared_ptr<PrimitiveType> type) {
- return NotImplemented("Deserialization of Literal is not implemented yet");
+ return Conversions::FromBytes(type, data);
Review Comment:
> Why wasn't the suggested code adopted? I share the same idea ;)
You're right. There have been quite a few comments in the review, so I missed this one. Thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2378654497
##########
src/iceberg/expression/literal.h:
##########
@@ -71,6 +71,7 @@ class ICEBERG_EXPORT Literal {
static Literal Double(double value);
static Literal String(std::string value);
static Literal Binary(std::vector<uint8_t> value);
+ static Literal Fixed(std::vector<uint8_t> value, int32_t length);
Review Comment:
This comment seems contradictions with the above？So should we use
```
Literal Literal::Fixed(std::vector<uint8_t> value) {
auto length = static_cast<int32_t>(value.size());
return {Value{std::move(value)}, fixed(length)};
}
``` to implement it, not use ICEBERG_DCHECK?
##########
src/iceberg/expression/literal.h:
##########
@@ -71,6 +71,7 @@ class ICEBERG_EXPORT Literal {
static Literal Double(double value);
static Literal String(std::string value);
static Literal Binary(std::vector<uint8_t> value);
+ static Literal Fixed(std::vector<uint8_t> value, int32_t length);
Review Comment:
This comment seems contradictions with the above？So should we use
```
Literal Literal::Fixed(std::vector<uint8_t> value) {
auto length = static_cast<int32_t>(value.size());
return {Value{std::move(value)}, fixed(length)};
}
```
to implement it, not use ICEBERG_DCHECK?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#issuecomment-3333486634
@wgtmac I've handled these comments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2295507915
##########
test/manifest_list_reader_test.cc:
##########
@@ -76,43 +77,40 @@ class ManifestListReaderV1Test : public ManifestListReaderTestBase {
std::vector<int64_t> file_size = {6185, 6113};
std::vector<int64_t> snapshot_id = {7532614258660258098, 7532614258660258098};
- std::vector<std::vector<std::uint8_t>> lower_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x30, 0x32, 0x2D, 0x32, 0x32},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x32}};
-
- std::vector<std::vector<std::uint8_t>> upper_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33}};
-
- return {{.manifest_path = paths[0],
- .manifest_length = file_size[0],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[0],
- .added_files_count = 4,
- .existing_files_count = 0,
- .deleted_files_count = 0,
- .added_rows_count = 6,
- .existing_rows_count = 0,
- .deleted_rows_count = 0,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[0],
- .upper_bound = upper_bounds[0]}}},
-
- {.manifest_path = paths[1],
- .manifest_length = file_size[1],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[1],
- .added_files_count = 0,
- .existing_files_count = 0,
- .deleted_files_count = 2,
- .added_rows_count = 0,
- .existing_rows_count = 0,
- .deleted_rows_count = 6,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[1],
- .upper_bound = upper_bounds[1]}}}};
+ return {
+ {.manifest_path = paths[0],
+ .manifest_length = file_size[0],
+ .partition_spec_id = 0,
+ .added_snapshot_id = snapshot_id[0],
+ .added_files_count = 4,
+ .existing_files_count = 0,
+ .deleted_files_count = 0,
+ .added_rows_count = 6,
+ .existing_rows_count = 0,
+ .deleted_rows_count = 0,
+ .partitions = {{.contains_null = false,
+ .contains_nan = false,
+ .lower_bound =
+ iceberg::Literal::String("2022-02-22").Serialize().value(),
+ .upper_bound =
+ iceberg::Literal::String("2022-2-23").Serialize().value()}}},
+
+ {.manifest_path = paths[1],
+ .manifest_length = file_size[1],
+ .partition_spec_id = 0,
+ .added_snapshot_id = snapshot_id[1],
+ .added_files_count = 0,
+ .existing_files_count = 0,
+ .deleted_files_count = 2,
+ .added_rows_count = 0,
+ .existing_rows_count = 0,
+ .deleted_rows_count = 6,
+ .partitions = {
+ {.contains_null = false,
+ .contains_nan = false,
+ .lower_bound = iceberg::Literal::String("2022-2-22").Serialize().value(),
Review Comment:
```suggestion
.lower_bound = Literal::String("2022-2-22").Serialize().value(),
```
##########
test/manifest_list_reader_test.cc:
##########
@@ -76,43 +77,40 @@ class ManifestListReaderV1Test : public ManifestListReaderTestBase {
std::vector<int64_t> file_size = {6185, 6113};
std::vector<int64_t> snapshot_id = {7532614258660258098, 7532614258660258098};
- std::vector<std::vector<std::uint8_t>> lower_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x30, 0x32, 0x2D, 0x32, 0x32},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x32}};
-
- std::vector<std::vector<std::uint8_t>> upper_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33}};
-
- return {{.manifest_path = paths[0],
- .manifest_length = file_size[0],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[0],
- .added_files_count = 4,
- .existing_files_count = 0,
- .deleted_files_count = 0,
- .added_rows_count = 6,
- .existing_rows_count = 0,
- .deleted_rows_count = 0,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[0],
- .upper_bound = upper_bounds[0]}}},
-
- {.manifest_path = paths[1],
- .manifest_length = file_size[1],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[1],
- .added_files_count = 0,
- .existing_files_count = 0,
- .deleted_files_count = 2,
- .added_rows_count = 0,
- .existing_rows_count = 0,
- .deleted_rows_count = 6,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[1],
- .upper_bound = upper_bounds[1]}}}};
+ return {
+ {.manifest_path = paths[0],
+ .manifest_length = file_size[0],
+ .partition_spec_id = 0,
+ .added_snapshot_id = snapshot_id[0],
+ .added_files_count = 4,
+ .existing_files_count = 0,
+ .deleted_files_count = 0,
+ .added_rows_count = 6,
+ .existing_rows_count = 0,
+ .deleted_rows_count = 0,
+ .partitions = {{.contains_null = false,
+ .contains_nan = false,
+ .lower_bound =
+ iceberg::Literal::String("2022-02-22").Serialize().value(),
+ .upper_bound =
+ iceberg::Literal::String("2022-2-23").Serialize().value()}}},
Review Comment:
```suggestion
Literal::String("2022-2-23").Serialize().value()}}},
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Boolean(value != 0x00);
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Int(value);
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Date(value);
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+
+ if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else {
+ const char* type_name = [type_id]() {
+ switch (type_id) {
+ case TypeId::kLong:
+ return "Long";
+ case TypeId::kTime:
+ return "Time";
+ case TypeId::kTimestamp:
+ return "Timestamp";
+ case TypeId::kTimestampTz:
+ return "TimestampTz";
+ default:
+ return "Unknown";
+ }
+ }();
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
+ data.size());
+ }
+
+ return CreateLongTypeLiteral(value, type_id);
+ }
+
+ case TypeId::kFloat: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Float(value);
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Double(static_cast<double>(float_value));
+ } else if (data.size() == 8) {
+ // Standard 8-byte double
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Double(double_value);
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::String(
+ std::string(reinterpret_cast<const char*>(data.data()), data.size()));
+ }
+
+ case TypeId::kBinary: {
+ return Literal::Binary(std::vector<uint8_t>(data.begin(), data.end()));
+ }
+
+ case TypeId::kFixed: {
+ if (data.size() == 16) {
+ std::array<uint8_t, 16> fixed_bytes;
+ std::ranges::copy(data, fixed_bytes.begin());
+ return Literal(Literal::Value{fixed_bytes}, type);
+ } else {
+ return Literal(Literal::Value{std::vector<uint8_t>(data.begin(), data.end())},
+ type);
+ }
+ }
+
+ case TypeId::kUuid: {
+ ICEBERG_ASSIGN_OR_RAISE(auto uuid_value, ReadBigEndian16(data));
+ return Literal(Literal::Value{uuid_value}, type);
+ }
+
+ case TypeId::kDecimal: {
+ if (data.size() > 16) {
+ return InvalidArgument(
+ "Decimal data too large, maximum 16 bytes supported, got {}", data.size());
+ }
+
+ std::array<uint8_t, 16> decimal_bytes{};
+ // Copy data to the end of the array (big-endian format for decimals)
+ // This handles variable-length decimals by right-aligning them
+ std::ranges::copy(data, decimal_bytes.end() - data.size());
+ return Literal(Literal::Value{decimal_bytes}, type);
+ }
+
+ default:
+ return NotImplemented("Deserialization for type {} is not supported",
+ type->ToString());
Review Comment:
```suggestion
return NotSupported("Cannot deserialize type: {}", type->ToString());
```
##########
src/iceberg/expression/literal.h:
##########
@@ -144,6 +144,19 @@ class ICEBERG_EXPORT Literal {
Literal(Value value, std::shared_ptr<PrimitiveType> type);
friend class LiteralCaster;
+ friend class LiteralSerializer;
+
+ /// \brief Format a date value as a string.
+ std::string FormatDate(int32_t days_since_epoch) const;
Review Comment:
These `FormatXXX` functions are used only internally, right? We should either move them to `literal.cc` or add a separate utility class for formatting time values like `iceberg/util/time_utils.h`. I prefer the latter because we can add test cases for them.
##########
test/manifest_list_reader_test.cc:
##########
@@ -76,43 +77,40 @@ class ManifestListReaderV1Test : public ManifestListReaderTestBase {
std::vector<int64_t> file_size = {6185, 6113};
std::vector<int64_t> snapshot_id = {7532614258660258098, 7532614258660258098};
- std::vector<std::vector<std::uint8_t>> lower_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x30, 0x32, 0x2D, 0x32, 0x32},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x32}};
-
- std::vector<std::vector<std::uint8_t>> upper_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33}};
-
- return {{.manifest_path = paths[0],
- .manifest_length = file_size[0],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[0],
- .added_files_count = 4,
- .existing_files_count = 0,
- .deleted_files_count = 0,
- .added_rows_count = 6,
- .existing_rows_count = 0,
- .deleted_rows_count = 0,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[0],
- .upper_bound = upper_bounds[0]}}},
-
- {.manifest_path = paths[1],
- .manifest_length = file_size[1],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[1],
- .added_files_count = 0,
- .existing_files_count = 0,
- .deleted_files_count = 2,
- .added_rows_count = 0,
- .existing_rows_count = 0,
- .deleted_rows_count = 6,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[1],
- .upper_bound = upper_bounds[1]}}}};
+ return {
+ {.manifest_path = paths[0],
+ .manifest_length = file_size[0],
+ .partition_spec_id = 0,
+ .added_snapshot_id = snapshot_id[0],
+ .added_files_count = 4,
+ .existing_files_count = 0,
+ .deleted_files_count = 0,
+ .added_rows_count = 6,
+ .existing_rows_count = 0,
+ .deleted_rows_count = 0,
+ .partitions = {{.contains_null = false,
+ .contains_nan = false,
+ .lower_bound =
+ iceberg::Literal::String("2022-02-22").Serialize().value(),
+ .upper_bound =
+ iceberg::Literal::String("2022-2-23").Serialize().value()}}},
+
+ {.manifest_path = paths[1],
+ .manifest_length = file_size[1],
+ .partition_spec_id = 0,
+ .added_snapshot_id = snapshot_id[1],
+ .added_files_count = 0,
+ .existing_files_count = 0,
+ .deleted_files_count = 2,
+ .added_rows_count = 0,
+ .existing_rows_count = 0,
+ .deleted_rows_count = 6,
+ .partitions = {
+ {.contains_null = false,
+ .contains_nan = false,
+ .lower_bound = iceberg::Literal::String("2022-2-22").Serialize().value(),
+ .upper_bound =
+ iceberg::Literal::String("2022-2-23").Serialize().value()}}}};
Review Comment:
```suggestion
Literal::String("2022-2-23").Serialize().value()}}}};
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
Review Comment:
```suggestion
return NotSupported("Cannot serialize BelowMin");
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
Review Comment:
Can we use C++20 concepts instead of `static_assert` which is more type-safe? We should only support limited set of types.
##########
test/manifest_list_reader_test.cc:
##########
@@ -76,43 +77,40 @@ class ManifestListReaderV1Test : public ManifestListReaderTestBase {
std::vector<int64_t> file_size = {6185, 6113};
std::vector<int64_t> snapshot_id = {7532614258660258098, 7532614258660258098};
- std::vector<std::vector<std::uint8_t>> lower_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x30, 0x32, 0x2D, 0x32, 0x32},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x32}};
-
- std::vector<std::vector<std::uint8_t>> upper_bounds = {
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33},
- {0x32, 0x30, 0x32, 0x32, 0x2D, 0x32, 0x2D, 0x32, 0x33}};
-
- return {{.manifest_path = paths[0],
- .manifest_length = file_size[0],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[0],
- .added_files_count = 4,
- .existing_files_count = 0,
- .deleted_files_count = 0,
- .added_rows_count = 6,
- .existing_rows_count = 0,
- .deleted_rows_count = 0,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[0],
- .upper_bound = upper_bounds[0]}}},
-
- {.manifest_path = paths[1],
- .manifest_length = file_size[1],
- .partition_spec_id = 0,
- .added_snapshot_id = snapshot_id[1],
- .added_files_count = 0,
- .existing_files_count = 0,
- .deleted_files_count = 2,
- .added_rows_count = 0,
- .existing_rows_count = 0,
- .deleted_rows_count = 6,
- .partitions = {{.contains_null = false,
- .contains_nan = false,
- .lower_bound = lower_bounds[1],
- .upper_bound = upper_bounds[1]}}}};
+ return {
+ {.manifest_path = paths[0],
+ .manifest_length = file_size[0],
+ .partition_spec_id = 0,
+ .added_snapshot_id = snapshot_id[0],
+ .added_files_count = 4,
+ .existing_files_count = 0,
+ .deleted_files_count = 0,
+ .added_rows_count = 6,
+ .existing_rows_count = 0,
+ .deleted_rows_count = 0,
+ .partitions = {{.contains_null = false,
+ .contains_nan = false,
+ .lower_bound =
+ iceberg::Literal::String("2022-02-22").Serialize().value(),
Review Comment:
```suggestion
Literal::String("2022-02-22").Serialize().value(),
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if (data.size() < sizeof(T)) {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ value = std::byteswap(value);
+ }
+ return value;
+}
+
+/// \brief Write a 16-byte value in big-endian format (for UUID and Decimal).
+void WriteBigEndian16(std::vector<uint8_t>& buffer,
+ const std::array<uint8_t, 16>& value) {
+ buffer.insert(buffer.end(), value.begin(), value.end());
+}
+
+/// \brief Read a 16-byte value in big-endian format (for UUID and Decimal).
+Result<std::array<uint8_t, 16>> ReadBigEndian16(std::span<const uint8_t> data) {
Review Comment:
Same question as above.
##########
test/manifest_reader_test.cc:
##########
@@ -59,24 +59,33 @@ class ManifestReaderV1Test : public TempFileTestBase {
"order_ts_hour=2021-01-26-00/"
"00000-2-d5ae78b7-4449-45ec-adb7-c0e9c0bdb714-0-00004.parquet"};
std::vector<int64_t> partitions = {447696, 473976, 465192, 447672};
+
+ // TODO(Li Feiyang): The Decimal type and its serialization logic are not yet fully
+ // implemented to support variable-length encoding as required by the Iceberg
+ // specification. Using Literal::Binary as a temporary substitute to represent the raw
+ // bytes for the decimal values.
std::vector<std::map<int32_t, std::vector<uint8_t>>> bounds = {
- {{1, {0xd2, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {'.', 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x12, 0xe2}},
- {4, {0xc0, 'y', 0xe7, 0x98, 0xd6, 0xb9, 0x05, 0x00}}},
- {{1, {0xd2, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {'.', 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x12, 0xe3}},
- {4, {0xc0, 0x19, '#', '=', 0xe2, 0x0f, 0x06, 0x00}}},
- {{1, {'{', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x0e, '"'}},
- {4, {0xc0, 0xd9, '7', 0x93, 0x1f, 0xf3, 0x05, 0x00}}},
- {{1, {'{', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x0e, '!'}},
- {4, {0xc0, 0x19, 0x10, '{', 0xc2, 0xb9, 0x05, 0x00}}},
+ {{1, iceberg::Literal::Long(1234).Serialize().value()},
Review Comment:
```suggestion
{{1, Literal::Long(1234).Serialize().value()},
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if (data.size() < sizeof(T)) {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ value = std::byteswap(value);
+ }
+ return value;
+}
+
+/// \brief Write a 16-byte value in big-endian format (for UUID and Decimal).
+void WriteBigEndian16(std::vector<uint8_t>& buffer,
Review Comment:
Is it possible to reuse the same template function signature as `WriteBigEndian` with specialization for `std::array<uint8_t, 16>`?
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
Review Comment:
I would recommend to add `iceberg/util/endian.h` like https://github.com/apache/arrow/blob/main/cpp/src/arrow/util/endian.h
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if (data.size() < sizeof(T)) {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ value = std::byteswap(value);
+ }
+ return value;
+}
+
+/// \brief Write a 16-byte value in big-endian format (for UUID and Decimal).
+void WriteBigEndian16(std::vector<uint8_t>& buffer,
+ const std::array<uint8_t, 16>& value) {
+ buffer.insert(buffer.end(), value.begin(), value.end());
+}
+
+/// \brief Read a 16-byte value in big-endian format (for UUID and Decimal).
+Result<std::array<uint8_t, 16>> ReadBigEndian16(std::span<const uint8_t> data) {
+ if (data.size() < 16) {
+ return InvalidArgument("Insufficient data to read 16 bytes, got {}", data.size());
+ }
+
+ std::array<uint8_t, 16> result;
+ std::copy(data.begin(), data.begin() + 16, result.begin());
+ return result;
+}
+
+/// \brief Create a literal from int64 value and long-type TypeId.
+Result<Literal> CreateLongTypeLiteral(int64_t value, TypeId type_id) {
+ switch (type_id) {
+ case TypeId::kLong:
+ return Literal::Long(value);
+ case TypeId::kTime:
+ return Literal::Time(value);
+ case TypeId::kTimestamp:
+ return Literal::Timestamp(value);
+ case TypeId::kTimestampTz:
+ return Literal::TimestampTz(value);
+ default:
+ return InvalidArgument("Unsupported long type: {}", static_cast<int>(type_id));
+ }
+}
+
+} // namespace
+
+/// \brief LiteralSerializer handles serialization/deserialization operations for Literal.
+/// This is an internal implementation class.
+class LiteralSerializer {
Review Comment:
Checking the java code, it seems that single-value serialization is not only used by Literal. We need to create a separate utility file to support reuse. For example, we can create `iceberg/util/conversions.h` and provide following functions:
```cpp
class ICEBERG_EXPORT Conversions {
public:
static Result<std::vector<uint8_t>> ToBytes(const Primitive& type, const Literal::Value& value);
static Result<std::vector<uint8_t>> ToBytes(const Literal& literal);
static Result<Literal::Value> FromBytes(const Primitive& type, std::span<const uint8_t> data);
static Result<Literal> FromBytes(std::shared_ptr<Primitive> type, std::span<const uint8_t> data);
};
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
Review Comment:
```suggestion
return NotSupported("Cannot serialize AboveMax");
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
Review Comment:
```suggestion
return NotSupported("Cannot serialize null");
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
Review Comment:
```suggestion
return NotSupported("Serialization for type {} is not supported",
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
Review Comment:
It shouldn't serialize null.
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Boolean(value != 0x00);
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Int(value);
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Date(value);
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+
+ if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else {
+ const char* type_name = [type_id]() {
+ switch (type_id) {
+ case TypeId::kLong:
+ return "Long";
+ case TypeId::kTime:
+ return "Time";
+ case TypeId::kTimestamp:
+ return "Timestamp";
+ case TypeId::kTimestampTz:
+ return "TimestampTz";
+ default:
+ return "Unknown";
+ }
+ }();
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
+ data.size());
+ }
+
+ return CreateLongTypeLiteral(value, type_id);
+ }
+
+ case TypeId::kFloat: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Float(value);
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Double(static_cast<double>(float_value));
+ } else if (data.size() == 8) {
+ // Standard 8-byte double
+ ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
+ return Literal::Double(double_value);
+ } else {
+ return InvalidArgument("Double requires 4 or 8 bytes, got {}", data.size());
+ }
+ }
+
+ case TypeId::kString: {
+ return Literal::String(
+ std::string(reinterpret_cast<const char*>(data.data()), data.size()));
+ }
+
+ case TypeId::kBinary: {
+ return Literal::Binary(std::vector<uint8_t>(data.begin(), data.end()));
+ }
+
+ case TypeId::kFixed: {
+ if (data.size() == 16) {
+ std::array<uint8_t, 16> fixed_bytes;
+ std::ranges::copy(data, fixed_bytes.begin());
+ return Literal(Literal::Value{fixed_bytes}, type);
+ } else {
+ return Literal(Literal::Value{std::vector<uint8_t>(data.begin(), data.end())},
+ type);
+ }
+ }
+
+ case TypeId::kUuid: {
+ ICEBERG_ASSIGN_OR_RAISE(auto uuid_value, ReadBigEndian16(data));
+ return Literal(Literal::Value{uuid_value}, type);
+ }
+
+ case TypeId::kDecimal: {
+ if (data.size() > 16) {
+ return InvalidArgument(
+ "Decimal data too large, maximum 16 bytes supported, got {}", data.size());
+ }
+
+ std::array<uint8_t, 16> decimal_bytes{};
+ // Copy data to the end of the array (big-endian format for decimals)
+ // This handles variable-length decimals by right-aligning them
+ std::ranges::copy(data, decimal_bytes.end() - data.size());
+ return Literal(Literal::Value{decimal_bytes}, type);
+ }
+
+ default:
+ return NotImplemented("Deserialization for type {} is not supported",
+ type->ToString());
+ }
+
+ // Should not reach here
+ return InvalidArgument("Unexpected error in deserialization");
Review Comment:
```suggestion
std::unreachable();
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Boolean(value != 0x00);
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Int(value);
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Date(value);
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+
+ if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else {
+ const char* type_name = [type_id]() {
+ switch (type_id) {
+ case TypeId::kLong:
+ return "Long";
+ case TypeId::kTime:
+ return "Time";
+ case TypeId::kTimestamp:
+ return "Timestamp";
+ case TypeId::kTimestampTz:
+ return "TimestampTz";
+ default:
+ return "Unknown";
+ }
+ }();
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
+ data.size());
+ }
+
+ return CreateLongTypeLiteral(value, type_id);
Review Comment:
```suggestion
return {value, std::move(type)};
```
Why not directly use `Literal(Value value, std::shared_ptr<PrimitiveType> type)` to create it? You don't need CreateLongTypeLiteral.
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Boolean(value != 0x00);
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Int(value);
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Date(value);
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+
+ if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else {
+ const char* type_name = [type_id]() {
Review Comment:
Add a `ICEBERG_EXPORT std::string_view ToString(TypeId id)` to `iceberg/util/type.h`? Please also use lowercase form like what we did in `Type::ToString()`.
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if (data.size() < sizeof(T)) {
Review Comment:
```suggestion
if (data.size() < sizeof(T)) [[unlikely]] {
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -294,13 +388,21 @@ std::string Literal::ToString() const {
}
return result;
}
+ case TypeId::kDate: {
+ return FormatDate(std::get<int32_t>(value_));
+ }
+ case TypeId::kTime: {
+ return FormatTime(std::get<int64_t>(value_));
+ }
+ case TypeId::kTimestamp: {
+ return FormatTimestamp(std::get<int64_t>(value_));
+ }
+ case TypeId::kTimestampTz: {
+ return FormatTimestampTz(std::get<int64_t>(value_));
+ }
case TypeId::kDecimal:
case TypeId::kUuid:
- case TypeId::kFixed:
- case TypeId::kDate:
- case TypeId::kTime:
- case TypeId::kTimestamp:
- case TypeId::kTimestampTz: {
+ case TypeId::kFixed: {
Review Comment:
BTW, please help me check the `ToString` should follow these rules:
1. String literals should be printed as is.
2. Binary and fixed literals should be printed as base16-encoded texts and quoted by `X''`.
3. Other values should be quoted in `""`.
I'd suggest to create a separate PR to fix this. WDYT?
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
Review Comment:
```suggestion
} else if constexpr (sizeof(T) > 1) {
T le_value = std::byteswap(value);
const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
} else {
// For single byte, no byteswap needed
buffer.push_back(static_cast<uint8_t>(value));
}
```
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Boolean(value != 0x00);
+ }
+
+ case TypeId::kInt: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Int(value);
+ }
+
+ case TypeId::kDate: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Date(value);
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+
+ if (data.size() == 4) {
+ // Type was promoted from int to long
+ ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
+ value = static_cast<int64_t>(int_value);
+ } else if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else {
+ const char* type_name = [type_id]() {
+ switch (type_id) {
+ case TypeId::kLong:
+ return "Long";
+ case TypeId::kTime:
+ return "Time";
+ case TypeId::kTimestamp:
+ return "Timestamp";
+ case TypeId::kTimestampTz:
+ return "TimestampTz";
+ default:
+ return "Unknown";
+ }
+ }();
+ return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
+ data.size());
+ }
+
+ return CreateLongTypeLiteral(value, type_id);
+ }
+
+ case TypeId::kFloat: {
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<float>(data));
+ return Literal::Float(value);
+ }
+
+ case TypeId::kDouble: {
+ if (data.size() == 4) {
+ // Type was promoted from float to double
+ ICEBERG_ASSIGN_OR_RAISE(auto float_value, ReadLittleEndian<float>(data));
+ return Literal::Double(static_cast<double>(float_value));
+ } else if (data.size() == 8) {
Review Comment:
Put `if (data.size() == 8)` before `if (data.size() == 4)`. The reason is that `8` should be more common than `4` in real cases.
##########
test/manifest_reader_test.cc:
##########
@@ -59,24 +59,33 @@ class ManifestReaderV1Test : public TempFileTestBase {
"order_ts_hour=2021-01-26-00/"
"00000-2-d5ae78b7-4449-45ec-adb7-c0e9c0bdb714-0-00004.parquet"};
std::vector<int64_t> partitions = {447696, 473976, 465192, 447672};
+
+ // TODO(Li Feiyang): The Decimal type and its serialization logic are not yet fully
+ // implemented to support variable-length encoding as required by the Iceberg
+ // specification. Using Literal::Binary as a temporary substitute to represent the raw
+ // bytes for the decimal values.
std::vector<std::map<int32_t, std::vector<uint8_t>>> bounds = {
- {{1, {0xd2, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {'.', 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x12, 0xe2}},
- {4, {0xc0, 'y', 0xe7, 0x98, 0xd6, 0xb9, 0x05, 0x00}}},
- {{1, {0xd2, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {'.', 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x12, 0xe3}},
- {4, {0xc0, 0x19, '#', '=', 0xe2, 0x0f, 0x06, 0x00}}},
- {{1, {'{', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x0e, '"'}},
- {4, {0xc0, 0xd9, '7', 0x93, 0x1f, 0xf3, 0x05, 0x00}}},
- {{1, {'{', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {2, {0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
- {3, {0x0e, '!'}},
- {4, {0xc0, 0x19, 0x10, '{', 0xc2, 0xb9, 0x05, 0x00}}},
+ {{1, iceberg::Literal::Long(1234).Serialize().value()},
Review Comment:
Same apply for below
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
Review Comment:
ditio, try to explore concepts.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2380927806
##########
test/literal_test.cc:
##########
@@ -383,4 +382,214 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
Review Comment:
```suggestion
class LiteralSerDeTest : public ::testing::TestWithParam<LiteralParam> {
```
##########
test/literal_test.cc:
##########
@@ -383,4 +382,214 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
Review Comment:
```suggestion
struct LiteralParam {
std::string test_name;
std::vector<uint8_t> serialized;
Literal value;
std::shared_ptr<PrimitiveType> type;
```
##########
test/literal_test.cc:
##########
@@ -383,4 +382,214 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
+};
+
+TEST_P(LiteralSerializationParam, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(*literal_result, param.expected_literal);
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(*final_literal, param.expected_literal);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerialization, LiteralSerializationParam,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "IntMaxValue", {255, 255, 255, 127}, Literal::Int(2147483647), int32()},
+ LiteralRoundTripParam{
+ "IntMinValue", {0, 0, 0, 128}, Literal::Int(-2147483648), int32()},
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"LongMaxValue",
+ {255, 255, 255, 255, 255, 255, 255, 127},
+ Literal::Long(std::numeric_limits<int64_t>::max()),
+ int64()},
+ LiteralRoundTripParam{"LongMinValue",
+ {0, 0, 0, 0, 0, 0, 0, 128},
+ Literal::Long(std::numeric_limits<int64_t>::min()),
+ int64()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{"FloatNegativeInfinity",
+ {0, 0, 128, 255},
+ Literal::Float(-std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatMaxValue",
+ {255, 255, 127, 127},
+ Literal::Float(std::numeric_limits<float>::max()),
+ float32()},
+ LiteralRoundTripParam{"FloatMinValue",
+ {255, 255, 127, 255},
+ Literal::Float(std::numeric_limits<float>::lowest()),
+ float32()},
+
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"DoubleNegativeInfinity",
+ {0, 0, 0, 0, 0, 0, 240, 255},
+ Literal::Double(-std::numeric_limits<double>::infinity()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMaxValue",
+ {255, 255, 255, 255, 255, 255, 239, 127},
+ Literal::Double(std::numeric_limits<double>::max()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMinValue",
+ {255, 255, 255, 255, 255, 255, 239, 255},
+ Literal::Double(std::numeric_limits<double>::lowest()),
+ float64()},
+
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{
+ "StringLong",
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ Literal::String("AAAAAAAAAAAAAAAA"),
+ string()},
+
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ LiteralRoundTripParam{"BinarySingleByte", {42}, Literal::Binary({42}), binary()},
+
+ // Fixed type
+ LiteralRoundTripParam{"FixedLength4",
+ {0x01, 0x02, 0x03, 0x04},
+ Literal::Fixed({0x01, 0x02, 0x03, 0x04}),
+ fixed(4)},
+ LiteralRoundTripParam{
+ "FixedLength8",
+ {0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11},
+ Literal::Fixed({0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11}),
+ fixed(8)},
+ LiteralRoundTripParam{
+ "FixedLength16",
+ {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F},
+ Literal::Fixed({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}),
+ fixed(16)},
+ LiteralRoundTripParam{
+ "FixedSingleByte", {0xFF}, Literal::Fixed({0xFF}), fixed(1)},
+
+ // Temporal types
+ LiteralRoundTripParam{"DateEpoch", {0, 0, 0, 0}, Literal::Date(0), date()},
+ LiteralRoundTripParam{"DateNextDay", {1, 0, 0, 0}, Literal::Date(1), date()},
+ LiteralRoundTripParam{"DateY2K", {205, 42, 0, 0}, Literal::Date(10957), date()},
+ LiteralRoundTripParam{
+ "DateNegative", {255, 255, 255, 255}, Literal::Date(-1), date()},
+
+ LiteralRoundTripParam{
+ "TimeMidnight", {0, 0, 0, 0, 0, 0, 0, 0}, Literal::Time(0), time()},
+ LiteralRoundTripParam{"TimeNoon",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Time(45045123456),
+ time()},
+ LiteralRoundTripParam{
+ "TimeOneSecond", {64, 66, 15, 0, 0, 0, 0, 0}, Literal::Time(1000000), time()},
+
+ LiteralRoundTripParam{"TimestampEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::Timestamp(0),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampOneSecond",
+ {64, 66, 15, 0, 0, 0, 0, 0},
+ Literal::Timestamp(1000000),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampNoon2024",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Timestamp(45045123456),
+ timestamp()},
+
+ LiteralRoundTripParam{"TimestampTzEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::TimestampTz(0),
+ timestamp_tz()},
+ LiteralRoundTripParam{"TimestampTzOneHour",
+ {0, 164, 147, 214, 0, 0, 0, 0},
+ Literal::TimestampTz(3600000000),
+ timestamp_tz()}),
+
+ [](const testing::TestParamInfo<LiteralSerializationParam::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationTest, EmptyString) {
Review Comment:
Shouldn't this be a case of the above test? For example:
```cpp
LiteralRoundTripParam{"EmptyString",
{},
Literal::String({""}),
string()},
```
##########
test/literal_test.cc:
##########
@@ -383,4 +382,214 @@ TEST(LiteralTest, DoubleZeroComparison) {
EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
}
+struct LiteralRoundTripParam {
+ std::string test_name;
+ std::vector<uint8_t> input_bytes;
+ Literal expected_literal;
+ std::shared_ptr<PrimitiveType> type;
+};
+
+class LiteralSerializationParam : public ::testing::TestWithParam<LiteralRoundTripParam> {
+};
+
+TEST_P(LiteralSerializationParam, RoundTrip) {
+ const auto& param = GetParam();
+
+ // Deserialize from bytes
+ Result<Literal> literal_result = Literal::Deserialize(param.input_bytes, param.type);
+ ASSERT_TRUE(literal_result.has_value())
+ << "Deserialization failed: " << literal_result.error().message;
+
+ // Check type and value
+ EXPECT_EQ(*literal_result, param.expected_literal);
+
+ // Serialize back to bytes
+ Result<std::vector<uint8_t>> bytes_result = literal_result->Serialize();
+ ASSERT_TRUE(bytes_result.has_value())
+ << "Serialization failed: " << bytes_result.error().message;
+ EXPECT_EQ(*bytes_result, param.input_bytes);
+
+ // Deserialize again to verify idempotency
+ Result<Literal> final_literal = Literal::Deserialize(*bytes_result, param.type);
+ ASSERT_TRUE(final_literal.has_value())
+ << "Final deserialization failed: " << final_literal.error().message;
+ EXPECT_EQ(*final_literal, param.expected_literal);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ BinarySerialization, LiteralSerializationParam,
+ ::testing::Values(
+ // Basic types
+ LiteralRoundTripParam{"BooleanTrue", {1}, Literal::Boolean(true), boolean()},
+ LiteralRoundTripParam{"BooleanFalse", {0}, Literal::Boolean(false), boolean()},
+
+ LiteralRoundTripParam{"Int", {32, 0, 0, 0}, Literal::Int(32), int32()},
+ LiteralRoundTripParam{
+ "IntMaxValue", {255, 255, 255, 127}, Literal::Int(2147483647), int32()},
+ LiteralRoundTripParam{
+ "IntMinValue", {0, 0, 0, 128}, Literal::Int(-2147483648), int32()},
+ LiteralRoundTripParam{
+ "NegativeInt", {224, 255, 255, 255}, Literal::Int(-32), int32()},
+
+ LiteralRoundTripParam{
+ "Long", {32, 0, 0, 0, 0, 0, 0, 0}, Literal::Long(32), int64()},
+ LiteralRoundTripParam{"LongMaxValue",
+ {255, 255, 255, 255, 255, 255, 255, 127},
+ Literal::Long(std::numeric_limits<int64_t>::max()),
+ int64()},
+ LiteralRoundTripParam{"LongMinValue",
+ {0, 0, 0, 0, 0, 0, 0, 128},
+ Literal::Long(std::numeric_limits<int64_t>::min()),
+ int64()},
+ LiteralRoundTripParam{"NegativeLong",
+ {224, 255, 255, 255, 255, 255, 255, 255},
+ Literal::Long(-32),
+ int64()},
+
+ LiteralRoundTripParam{"Float", {0, 0, 128, 63}, Literal::Float(1.0f), float32()},
+ LiteralRoundTripParam{"FloatNegativeInfinity",
+ {0, 0, 128, 255},
+ Literal::Float(-std::numeric_limits<float>::infinity()),
+ float32()},
+ LiteralRoundTripParam{"FloatMaxValue",
+ {255, 255, 127, 127},
+ Literal::Float(std::numeric_limits<float>::max()),
+ float32()},
+ LiteralRoundTripParam{"FloatMinValue",
+ {255, 255, 127, 255},
+ Literal::Float(std::numeric_limits<float>::lowest()),
+ float32()},
+
+ LiteralRoundTripParam{
+ "Double", {0, 0, 0, 0, 0, 0, 240, 63}, Literal::Double(1.0), float64()},
+ LiteralRoundTripParam{"DoubleNegativeInfinity",
+ {0, 0, 0, 0, 0, 0, 240, 255},
+ Literal::Double(-std::numeric_limits<double>::infinity()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMaxValue",
+ {255, 255, 255, 255, 255, 255, 239, 127},
+ Literal::Double(std::numeric_limits<double>::max()),
+ float64()},
+ LiteralRoundTripParam{"DoubleMinValue",
+ {255, 255, 255, 255, 255, 255, 239, 255},
+ Literal::Double(std::numeric_limits<double>::lowest()),
+ float64()},
+
+ LiteralRoundTripParam{"String",
+ {105, 99, 101, 98, 101, 114, 103},
+ Literal::String("iceberg"),
+ string()},
+ LiteralRoundTripParam{
+ "StringLong",
+ {65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65},
+ Literal::String("AAAAAAAAAAAAAAAA"),
+ string()},
+
+ LiteralRoundTripParam{"BinaryData",
+ {0x01, 0x02, 0x03, 0xFF},
+ Literal::Binary({0x01, 0x02, 0x03, 0xFF}),
+ binary()},
+ LiteralRoundTripParam{"BinarySingleByte", {42}, Literal::Binary({42}), binary()},
+
+ // Fixed type
+ LiteralRoundTripParam{"FixedLength4",
+ {0x01, 0x02, 0x03, 0x04},
+ Literal::Fixed({0x01, 0x02, 0x03, 0x04}),
+ fixed(4)},
+ LiteralRoundTripParam{
+ "FixedLength8",
+ {0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11},
+ Literal::Fixed({0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11}),
+ fixed(8)},
+ LiteralRoundTripParam{
+ "FixedLength16",
+ {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F},
+ Literal::Fixed({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}),
+ fixed(16)},
+ LiteralRoundTripParam{
+ "FixedSingleByte", {0xFF}, Literal::Fixed({0xFF}), fixed(1)},
+
+ // Temporal types
+ LiteralRoundTripParam{"DateEpoch", {0, 0, 0, 0}, Literal::Date(0), date()},
+ LiteralRoundTripParam{"DateNextDay", {1, 0, 0, 0}, Literal::Date(1), date()},
+ LiteralRoundTripParam{"DateY2K", {205, 42, 0, 0}, Literal::Date(10957), date()},
+ LiteralRoundTripParam{
+ "DateNegative", {255, 255, 255, 255}, Literal::Date(-1), date()},
+
+ LiteralRoundTripParam{
+ "TimeMidnight", {0, 0, 0, 0, 0, 0, 0, 0}, Literal::Time(0), time()},
+ LiteralRoundTripParam{"TimeNoon",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Time(45045123456),
+ time()},
+ LiteralRoundTripParam{
+ "TimeOneSecond", {64, 66, 15, 0, 0, 0, 0, 0}, Literal::Time(1000000), time()},
+
+ LiteralRoundTripParam{"TimestampEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::Timestamp(0),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampOneSecond",
+ {64, 66, 15, 0, 0, 0, 0, 0},
+ Literal::Timestamp(1000000),
+ timestamp()},
+ LiteralRoundTripParam{"TimestampNoon2024",
+ {128, 9, 230, 124, 10, 0, 0, 0},
+ Literal::Timestamp(45045123456),
+ timestamp()},
+
+ LiteralRoundTripParam{"TimestampTzEpoch",
+ {0, 0, 0, 0, 0, 0, 0, 0},
+ Literal::TimestampTz(0),
+ timestamp_tz()},
+ LiteralRoundTripParam{"TimestampTzOneHour",
+ {0, 164, 147, 214, 0, 0, 0, 0},
+ Literal::TimestampTz(3600000000),
+ timestamp_tz()}),
+
+ [](const testing::TestParamInfo<LiteralSerializationParam::ParamType>& info) {
+ return info.param.test_name;
+ });
+
+TEST(LiteralSerializationTest, EmptyString) {
+ auto empty_string = Literal::String("");
+ auto empty_bytes = empty_string.Serialize();
+ ASSERT_TRUE(empty_bytes.has_value());
+ EXPECT_TRUE(empty_bytes->empty());
+
+ auto deserialize_result = Literal::Deserialize(*empty_bytes, string());
+ ASSERT_THAT(deserialize_result, IsOk());
+ EXPECT_TRUE(std::get<std::string>(deserialize_result->value()).empty());
+}
+
+TEST(LiteralSerializationTest, EmptyBinary) {
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2380702420
##########
src/iceberg/expression/literal.h:
##########
@@ -143,11 +144,76 @@ class ICEBERG_EXPORT Literal {
private:
Literal(Value value, std::shared_ptr<PrimitiveType> type);
+ friend class Conversions;
friend class LiteralCaster;
- private:
Review Comment:
There is another one above. I guess this is repeated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2297376271
##########
src/iceberg/expression/literal.cc:
##########
@@ -355,4 +457,305 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
target_type->ToString());
}
+// LiteralSerializer implementation
+
+Result<std::vector<uint8_t>> LiteralSerializer::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return InvalidArgument("Cannot serialize AboveMax literal");
+ }
+ if (literal.IsBelowMin()) {
+ return InvalidArgument("Cannot serialize BelowMin literal");
+ }
+
+ std::vector<uint8_t> result;
+
+ // Null values serialize to empty buffer
+ if (literal.IsNull()) {
+ return result;
+ }
+
+ const auto& value = literal.value();
+ const auto type_id = literal.type()->type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result = binary_data;
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ return InvalidArgument("Invalid value type for Fixed literal");
+ }
+ return result;
+ }
+
+ case TypeId::kUuid: {
+ // 16-byte big-endian value
+ const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
+ WriteBigEndian16(result, uuid_bytes);
+ return result;
+ }
+
+ default:
+ return NotImplemented("Serialization for type {} is not supported",
+ literal.type()->ToString());
+ }
+}
+
+Result<Literal> LiteralSerializer::FromBytes(std::span<const uint8_t> data,
+ std::shared_ptr<PrimitiveType> type) {
+ if (!type) {
+ return InvalidArgument("Type cannot be null");
+ }
+
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Null(type);
+ }
+
+ const auto type_id = type->type_id();
+
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2315236914
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
Review Comment:
> BTW, why not `std::vector<uint8_t> WriteLittleEndian(T value)`?
I think the current `void` signature treats the `std::vector<uint8_t>&` buffer as a write target (a sink), allowing data to be appended multiple times efficiently. In contrast, returning a vector would create a new std::vector object on the heap with each call, making the existing approach superior?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "HeartLinked (via GitHub)" <gi...@apache.org>.
HeartLinked commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2315374240
##########
src/iceberg/util/conversions.cc:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/conversions.h"
+
+#include <array>
+#include <cstring>
+#include <span>
+#include <string>
+
+#include "iceberg/util/endian.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+/// \brief Write a value in little-endian format to the buffer.
+template <EndianConvertible T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ value = ToLittleEndian(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <EndianConvertible T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ if (data.size() < sizeof(T)) [[unlikely]] {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+ return FromLittleEndian(value);
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
+ const Literal::Value& value) {
+ std::vector<uint8_t> result;
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ // 0x00 for false, 0x01 for true
+ result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
+ return result;
+ }
+
+ case TypeId::kInt: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kDate: {
+ // Stores days from 1970-01-01 in a 4-byte little-endian int
+ WriteLittleEndian(result, std::get<int32_t>(value));
+ return result;
+ }
+
+ case TypeId::kLong: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTime: {
+ // Stores microseconds from midnight in an 8-byte little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestamp: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
+ // long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kTimestampTz: {
+ // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
+ // little-endian long
+ WriteLittleEndian(result, std::get<int64_t>(value));
+ return result;
+ }
+
+ case TypeId::kFloat: {
+ // Stored as 4-byte little-endian
+ WriteLittleEndian(result, std::get<float>(value));
+ return result;
+ }
+
+ case TypeId::kDouble: {
+ // Stored as 8-byte little-endian
+ WriteLittleEndian(result, std::get<double>(value));
+ return result;
+ }
+
+ case TypeId::kString: {
+ // UTF-8 bytes (without length)
+ const auto& str = std::get<std::string>(value);
+ result.insert(result.end(), str.begin(), str.end());
+ return result;
+ }
+
+ case TypeId::kBinary: {
+ // Binary value (without length)
+ const auto& binary_data = std::get<std::vector<uint8_t>>(value);
+ result.insert(result.end(), binary_data.begin(), binary_data.end());
+ return result;
+ }
+
+ case TypeId::kFixed: {
+ // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
+ // std::vector<uint8_t>
+ if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
+ const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
+ result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
+ } else if (std::holds_alternative<std::vector<uint8_t>>(value)) {
+ result = std::get<std::vector<uint8_t>>(value);
+ } else {
+ std::string actual_type = std::visit(
+ [](auto&& arg) -> std::string { return typeid(arg).name(); }, value);
+
+ return InvalidArgument("Invalid value type for Fixed literal, got type: {}",
+ actual_type);
+ }
+ return result;
+ }
+ // TODO(Li Feiyang): Add support for UUID and Decimal
+
+ default:
+ return NotSupported("Serialization for type {} is not supported", type.ToString());
+ }
+}
+
+Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
+ // Cannot serialize special values
+ if (literal.IsAboveMax()) {
+ return NotSupported("Cannot serialize AboveMax");
+ }
+ if (literal.IsBelowMin()) {
+ return NotSupported("Cannot serialize BelowMin");
+ }
+ if (literal.IsNull()) {
+ return NotSupported("Cannot serialize null");
+ }
+
+ return ToBytes(*literal.type(), literal.value());
+}
+
+Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
+ std::span<const uint8_t> data) {
+ // Empty data represents null value
+ if (data.empty()) {
+ return Literal::Value{std::monostate{}};
+ }
+
+ const auto type_id = type.type_id();
+
+ switch (type_id) {
+ case TypeId::kBoolean: {
+ if (data.size() != 1) {
+ return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
+ // 0x00 for false, non-zero byte for true
+ return Literal::Value{static_cast<bool>(value != 0x00)};
+ }
+
+ case TypeId::kInt: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Int requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kDate: {
+ if (data.size() != sizeof(int32_t)) {
+ return InvalidArgument("Date requires {} bytes, got {}", sizeof(int32_t),
+ data.size());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<int32_t>(data));
+ return Literal::Value{value};
+ }
+
+ case TypeId::kLong:
+ case TypeId::kTime:
+ case TypeId::kTimestamp:
+ case TypeId::kTimestampTz: {
+ int64_t value;
+ if (data.size() == 8) {
+ // Standard 8-byte long
+ ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
+ value = long_value;
+ } else if (data.size() == 4) {
Review Comment:
no. All these 4 types should be int64_t.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: implement literal expressions with binary serialization support [iceberg-cpp]
Posted by "mapleFU (via GitHub)" <gi...@apache.org>.
mapleFU commented on code in PR #185:
URL: https://github.com/apache/iceberg-cpp/pull/185#discussion_r2296087426
##########
src/iceberg/expression/literal.cc:
##########
@@ -19,13 +19,107 @@
#include "iceberg/expression/literal.h"
+#include <algorithm>
+#include <bit>
+#include <chrono>
#include <cmath>
#include <concepts>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
#include "iceberg/exception.h"
+#include "iceberg/util/macros.h"
namespace iceberg {
+namespace {
+/// \brief Write a value in little-endian format to the buffer.
+template <typename T>
+void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if constexpr (std::endian::native == std::endian::little) {
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ if constexpr (sizeof(T) > 1) {
+ T le_value = std::byteswap(value);
+ const auto* bytes = reinterpret_cast<const uint8_t*>(&le_value);
+ buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+ } else {
+ // For single byte types, no byteswap needed
+ buffer.push_back(static_cast<uint8_t>(value));
+ }
+ }
+}
+
+/// \brief Read a value in little-endian format from the data.
+template <typename T>
+Result<T> ReadLittleEndian(std::span<const uint8_t> data) {
+ static_assert(std::is_trivially_copyable_v<T>, "Type must be trivially copyable");
+
+ if (data.size() < sizeof(T)) {
+ return InvalidArgument("Insufficient data to read {} bytes, got {}", sizeof(T),
+ data.size());
+ }
+
+ T value;
+ std::memcpy(&value, data.data(), sizeof(T));
+
+ if constexpr (std::endian::native != std::endian::little && sizeof(T) > 1) {
+ value = std::byteswap(value);
+ }
+ return value;
+}
+
+/// \brief Write a 16-byte value in big-endian format (for UUID and Decimal).
+void WriteBigEndian16(std::vector<uint8_t>& buffer,
Review Comment:
In C, the abi can pass 16bit argument in registers under linux platform, so generally small value would be better when passing by value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org