You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ga...@apache.org on 2026/05/05 16:16:23 UTC
(iceberg-cpp) branch main updated: feat: add iceberg_data library alongside iceberg (#631)
This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 0db527ae feat: add iceberg_data library alongside iceberg (#631)
0db527ae is described below
commit 0db527aef727bf98c699fd8bf930060a6fecf899
Author: Junwang Zhao <zh...@gmail.com>
AuthorDate: Wed May 6 00:16:17 2026 +0800
feat: add iceberg_data library alongside iceberg (#631)
Move data writers, deletes/, and puffin/ into a separate `iceberg_data`
library that links the existing `iceberg` target. `delete_file_index`
stays in `iceberg` because manifest_group embeds
DeleteFileIndex::Builder with only core dependencies.
* `iceberg` — unchanged target name for metadata/planning, expressions,
manifests, catalog (incl. in-memory), utilities, file I/O abstractions,
and delete_file_index.
* `iceberg_data` — data/, deletes/, puffin/; links `iceberg`.
`iceberg_bundle` links `iceberg_data` when the bundle is built.
`iceberg_rest` links `iceberg` and cpr only.
---
example/CMakeLists.txt | 2 +-
src/iceberg/CMakeLists.txt | 70 ++++++++++++++++------
src/iceberg/arrow_c_data_guard_internal.h | 9 +--
src/iceberg/data/data_writer.h | 6 +-
src/iceberg/data/delete_loader.h | 4 +-
src/iceberg/data/equality_delete_writer.h | 6 +-
.../CMakeLists.txt => src/iceberg/data/meson.build | 23 ++++---
src/iceberg/data/position_delete_writer.h | 6 +-
src/iceberg/data/writer.h | 6 +-
src/iceberg/deletes/position_delete_index.h | 4 +-
src/iceberg/deletes/roaring_position_bitmap.h | 4 +-
src/iceberg/file_writer.h | 2 +-
src/iceberg/iceberg-config.cmake.in | 2 +
...data_guard_internal.h => iceberg_data_export.h} | 56 ++++-------------
src/iceberg/meson.build | 54 +++++++++++++----
src/iceberg/puffin/file_metadata.h | 20 +++----
src/iceberg/puffin/json_serde_internal.h | 16 ++---
src/iceberg/puffin/puffin_format.h | 8 +--
src/iceberg/test/CMakeLists.txt | 15 +++--
src/iceberg/test/meson.build | 15 +++--
src/iceberg/test/roaring_test.cc | 60 -------------------
21 files changed, 188 insertions(+), 200 deletions(-)
diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt
index 0e04acce..f89058ed 100644
--- a/example/CMakeLists.txt
+++ b/example/CMakeLists.txt
@@ -22,7 +22,7 @@ project(example)
set(CMAKE_CXX_STANDARD 23)
-find_package(iceberg CONFIG REQUIRED)
+find_package(iceberg CONFIG REQUIRED COMPONENTS bundle rest)
add_executable(demo_example demo_example.cc)
diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt
index 62882d58..6d190cb6 100644
--- a/src/iceberg/CMakeLists.txt
+++ b/src/iceberg/CMakeLists.txt
@@ -20,14 +20,7 @@ set(ICEBERG_INCLUDES "$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>"
set(ICEBERG_SOURCES
arrow_c_data_guard_internal.cc
catalog/memory/in_memory_catalog.cc
- data/data_writer.cc
- data/delete_loader.cc
- data/equality_delete_writer.cc
- data/position_delete_writer.cc
- data/writer.cc
delete_file_index.cc
- deletes/roaring_position_bitmap.cc
- deletes/position_delete_index.cc
expression/aggregate.cc
expression/binder.cc
expression/evaluator.cc
@@ -66,9 +59,6 @@ set(ICEBERG_SOURCES
partition_field.cc
partition_spec.cc
partition_summary.cc
- puffin/file_metadata.cc
- puffin/puffin_format.cc
- puffin/json_serde.cc
row/arrow_array_wrapper.cc
row/manifest_wrapper.cc
row/partition_values.cc
@@ -134,24 +124,22 @@ list(APPEND
ICEBERG_STATIC_BUILD_INTERFACE_LIBS
"$<IF:$<BOOL:${NANOARROW_VENDORED}>,nanoarrow::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_static>,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>"
nlohmann_json::nlohmann_json
- roaring::roaring
ZLIB::ZLIB)
list(APPEND
ICEBERG_SHARED_BUILD_INTERFACE_LIBS
"$<IF:$<BOOL:${NANOARROW_VENDORED}>,nanoarrow::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_shared>,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>"
nlohmann_json::nlohmann_json
- roaring::roaring
ZLIB::ZLIB)
list(APPEND
ICEBERG_STATIC_INSTALL_INTERFACE_LIBS
"$<IF:$<BOOL:${NANOARROW_VENDORED}>,iceberg::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_static>,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>"
"$<IF:$<BOOL:${NLOHMANN_JSON_VENDORED}>,iceberg::nlohmann_json,$<IF:$<TARGET_EXISTS:nlohmann_json::nlohmann_json>,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>"
- "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+)
list(APPEND
ICEBERG_SHARED_INSTALL_INTERFACE_LIBS
"$<IF:$<BOOL:${NANOARROW_VENDORED}>,iceberg::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_shared>,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>"
"$<IF:$<BOOL:${NLOHMANN_JSON_VENDORED}>,iceberg::nlohmann_json,$<IF:$<TARGET_EXISTS:nlohmann_json::nlohmann_json>,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>"
- "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+)
add_iceberg_lib(iceberg
SOURCES
@@ -169,6 +157,52 @@ add_iceberg_lib(iceberg
OUTPUTS
ICEBERG_LIBRARIES)
+set(ICEBERG_DATA_SOURCES
+ data/data_writer.cc
+ data/delete_loader.cc
+ data/equality_delete_writer.cc
+ data/position_delete_writer.cc
+ data/writer.cc
+ deletes/position_delete_index.cc
+ deletes/roaring_position_bitmap.cc
+ puffin/file_metadata.cc
+ puffin/json_serde.cc
+ puffin/puffin_format.cc)
+
+set(ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS)
+set(ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS)
+set(ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS)
+set(ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS)
+
+list(APPEND ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS
+ "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
+ roaring::roaring)
+list(APPEND ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS
+ "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
+ roaring::roaring)
+list(APPEND
+ ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS
+ "$<IF:$<TARGET_EXISTS:iceberg::iceberg_static>,iceberg::iceberg_static,iceberg::iceberg_shared>"
+ "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+list(APPEND
+ ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS
+ "$<IF:$<TARGET_EXISTS:iceberg::iceberg_shared>,iceberg::iceberg_shared,iceberg::iceberg_static>"
+ "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+
+add_iceberg_lib(iceberg_data
+ SOURCES
+ ${ICEBERG_DATA_SOURCES}
+ EXTRA_INCLUDES
+ ${ICEBERG_INCLUDES}
+ SHARED_LINK_LIBS
+ ${ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS}
+ STATIC_LINK_LIBS
+ ${ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS}
+ STATIC_INSTALL_INTERFACE_LIBS
+ ${ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS}
+ SHARED_INSTALL_INTERFACE_LIBS
+ ${ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS})
+
iceberg_install_all_headers(iceberg)
add_subdirectory(catalog)
@@ -210,14 +244,14 @@ if(ICEBERG_BUILD_BUNDLE)
list(APPEND
ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
- "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
+ "$<IF:$<TARGET_EXISTS:iceberg_data_static>,iceberg_data_static,iceberg_data_shared>"
"$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>"
"$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>"
"$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_static>,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>"
)
list(APPEND
ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
- "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
+ "$<IF:$<TARGET_EXISTS:iceberg_data_shared>,iceberg_data_shared,iceberg_data_static>"
"$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>"
"$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>"
"$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>"
@@ -225,14 +259,14 @@ if(ICEBERG_BUILD_BUNDLE)
list(APPEND
ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
- "$<IF:$<TARGET_EXISTS:iceberg::iceberg_static>,iceberg::iceberg_static,iceberg::iceberg_shared>"
+ "$<IF:$<TARGET_EXISTS:iceberg::iceberg_data_static>,iceberg::iceberg_data_static,iceberg::iceberg_data_shared>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>>"
"$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_static>,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>>"
)
list(APPEND
ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS
- "$<IF:$<TARGET_EXISTS:iceberg::iceberg_shared>,iceberg::iceberg_shared,iceberg::iceberg_static>"
+ "$<IF:$<TARGET_EXISTS:iceberg::iceberg_data_shared>,iceberg::iceberg_data_shared,iceberg::iceberg_data_static>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>>"
"$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>"
diff --git a/src/iceberg/arrow_c_data_guard_internal.h b/src/iceberg/arrow_c_data_guard_internal.h
index 8bce14e5..04603b0f 100644
--- a/src/iceberg/arrow_c_data_guard_internal.h
+++ b/src/iceberg/arrow_c_data_guard_internal.h
@@ -22,10 +22,11 @@
#include <nanoarrow/nanoarrow.h>
#include "iceberg/arrow_c_data.h"
+#include "iceberg/iceberg_export.h"
namespace iceberg::internal {
-class ArrowArrayGuard {
+class ICEBERG_EXPORT ArrowArrayGuard {
public:
explicit ArrowArrayGuard(ArrowArray* array) : array_(array) {}
~ArrowArrayGuard();
@@ -34,7 +35,7 @@ class ArrowArrayGuard {
ArrowArray* array_;
};
-class ArrowSchemaGuard {
+class ICEBERG_EXPORT ArrowSchemaGuard {
public:
explicit ArrowSchemaGuard(ArrowSchema* schema) : schema_(schema) {}
~ArrowSchemaGuard();
@@ -43,7 +44,7 @@ class ArrowSchemaGuard {
ArrowSchema* schema_;
};
-class ArrowArrayViewGuard {
+class ICEBERG_EXPORT ArrowArrayViewGuard {
public:
explicit ArrowArrayViewGuard(ArrowArrayView* view) : view_(view) {}
~ArrowArrayViewGuard();
@@ -52,7 +53,7 @@ class ArrowArrayViewGuard {
ArrowArrayView* view_;
};
-class ArrowArrayBufferGuard {
+class ICEBERG_EXPORT ArrowArrayBufferGuard {
public:
explicit ArrowArrayBufferGuard(ArrowBuffer* buffer) : buffer_(buffer) {}
~ArrowArrayBufferGuard();
diff --git a/src/iceberg/data/data_writer.h b/src/iceberg/data/data_writer.h
index 380c97e2..9a2002f0 100644
--- a/src/iceberg/data/data_writer.h
+++ b/src/iceberg/data/data_writer.h
@@ -31,7 +31,7 @@
#include "iceberg/arrow_c_data.h"
#include "iceberg/data/writer.h"
#include "iceberg/file_format.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
#include "iceberg/row/partition_values.h"
#include "iceberg/type_fwd.h"
@@ -39,7 +39,7 @@
namespace iceberg {
/// \brief Options for creating a DataWriter.
-struct ICEBERG_EXPORT DataWriterOptions {
+struct ICEBERG_DATA_EXPORT DataWriterOptions {
std::string path;
std::shared_ptr<Schema> schema;
std::shared_ptr<PartitionSpec> spec;
@@ -51,7 +51,7 @@ struct ICEBERG_EXPORT DataWriterOptions {
};
/// \brief Writer for Iceberg data files.
-class ICEBERG_EXPORT DataWriter : public FileWriter {
+class ICEBERG_DATA_EXPORT DataWriter : public FileWriter {
public:
~DataWriter() override;
diff --git a/src/iceberg/data/delete_loader.h b/src/iceberg/data/delete_loader.h
index 0a1122f9..a422f068 100644
--- a/src/iceberg/data/delete_loader.h
+++ b/src/iceberg/data/delete_loader.h
@@ -26,14 +26,14 @@
#include <span>
#include <string_view>
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
#include "iceberg/type_fwd.h"
namespace iceberg {
/// \brief Loads delete files and constructs in-memory delete indexes.
-class ICEBERG_EXPORT DeleteLoader {
+class ICEBERG_DATA_EXPORT DeleteLoader {
public:
/// \brief Create a DeleteLoader.
/// \param io FileIO instance for reading delete files
diff --git a/src/iceberg/data/equality_delete_writer.h b/src/iceberg/data/equality_delete_writer.h
index d1728a48..fc0083a8 100644
--- a/src/iceberg/data/equality_delete_writer.h
+++ b/src/iceberg/data/equality_delete_writer.h
@@ -32,7 +32,7 @@
#include "iceberg/arrow_c_data.h"
#include "iceberg/data/writer.h"
#include "iceberg/file_format.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
#include "iceberg/row/partition_values.h"
#include "iceberg/type_fwd.h"
@@ -40,7 +40,7 @@
namespace iceberg {
/// \brief Options for creating an EqualityDeleteWriter.
-struct ICEBERG_EXPORT EqualityDeleteWriterOptions {
+struct ICEBERG_DATA_EXPORT EqualityDeleteWriterOptions {
std::string path;
std::shared_ptr<Schema> schema;
std::shared_ptr<PartitionSpec> spec;
@@ -54,7 +54,7 @@ struct ICEBERG_EXPORT EqualityDeleteWriterOptions {
};
/// \brief Writer for Iceberg equality delete files.
-class ICEBERG_EXPORT EqualityDeleteWriter : public FileWriter {
+class ICEBERG_DATA_EXPORT EqualityDeleteWriter : public FileWriter {
public:
~EqualityDeleteWriter() override;
diff --git a/example/CMakeLists.txt b/src/iceberg/data/meson.build
similarity index 67%
copy from example/CMakeLists.txt
copy to src/iceberg/data/meson.build
index 0e04acce..0f68decc 100644
--- a/example/CMakeLists.txt
+++ b/src/iceberg/data/meson.build
@@ -15,16 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-# Please set CMAKE_PREFIX_PATH to the install prefix of iceberg.
-cmake_minimum_required(VERSION 3.25)
-
-project(example)
-
-set(CMAKE_CXX_STANDARD 23)
-
-find_package(iceberg CONFIG REQUIRED)
-
-add_executable(demo_example demo_example.cc)
-
-target_link_libraries(demo_example PRIVATE iceberg::iceberg_bundle_shared
- iceberg::iceberg_rest_shared)
+install_headers(
+ [
+ 'data_writer.h',
+ 'delete_loader.h',
+ 'equality_delete_writer.h',
+ 'position_delete_writer.h',
+ 'writer.h',
+ ],
+ subdir: 'iceberg/data',
+)
diff --git a/src/iceberg/data/position_delete_writer.h b/src/iceberg/data/position_delete_writer.h
index de7a2c3f..c76e6172 100644
--- a/src/iceberg/data/position_delete_writer.h
+++ b/src/iceberg/data/position_delete_writer.h
@@ -31,7 +31,7 @@
#include "iceberg/arrow_c_data.h"
#include "iceberg/data/writer.h"
#include "iceberg/file_format.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
#include "iceberg/row/partition_values.h"
#include "iceberg/type_fwd.h"
@@ -39,7 +39,7 @@
namespace iceberg {
/// \brief Options for creating a PositionDeleteWriter.
-struct ICEBERG_EXPORT PositionDeleteWriterOptions {
+struct ICEBERG_DATA_EXPORT PositionDeleteWriterOptions {
std::string path;
std::shared_ptr<Schema> schema;
std::shared_ptr<PartitionSpec> spec;
@@ -51,7 +51,7 @@ struct ICEBERG_EXPORT PositionDeleteWriterOptions {
};
/// \brief Writer for Iceberg position delete files.
-class ICEBERG_EXPORT PositionDeleteWriter : public FileWriter {
+class ICEBERG_DATA_EXPORT PositionDeleteWriter : public FileWriter {
public:
~PositionDeleteWriter() override;
diff --git a/src/iceberg/data/writer.h b/src/iceberg/data/writer.h
index 82c1d0cd..a6f5f063 100644
--- a/src/iceberg/data/writer.h
+++ b/src/iceberg/data/writer.h
@@ -27,14 +27,14 @@
#include <vector>
#include "iceberg/arrow_c_data.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
#include "iceberg/type_fwd.h"
namespace iceberg {
/// \brief Base interface for data file writers.
-class ICEBERG_EXPORT FileWriter {
+class ICEBERG_DATA_EXPORT FileWriter {
public:
virtual ~FileWriter();
@@ -49,7 +49,7 @@ class ICEBERG_EXPORT FileWriter {
virtual Status Close() = 0;
/// \brief File metadata for all files produced by this writer.
- struct ICEBERG_EXPORT WriteResult {
+ struct ICEBERG_DATA_EXPORT WriteResult {
/// Usually a writer produces a single data or delete file.
/// Position delete writer may produce multiple file-scoped delete files.
/// In the future, multiple files can be produced if file rolling is supported.
diff --git a/src/iceberg/deletes/position_delete_index.h b/src/iceberg/deletes/position_delete_index.h
index 968d3ebe..5de82a59 100644
--- a/src/iceberg/deletes/position_delete_index.h
+++ b/src/iceberg/deletes/position_delete_index.h
@@ -26,7 +26,7 @@
#include <memory>
#include "iceberg/deletes/roaring_position_bitmap.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
namespace iceberg {
@@ -35,7 +35,7 @@ namespace iceberg {
/// This class provides a domain-specific API for position deletes
/// in Iceberg MOR (merge-on-read) tables. Positions are 0-based
/// row indices within a data file.
-class ICEBERG_EXPORT PositionDeleteIndex {
+class ICEBERG_DATA_EXPORT PositionDeleteIndex {
public:
PositionDeleteIndex() = default;
~PositionDeleteIndex() = default;
diff --git a/src/iceberg/deletes/roaring_position_bitmap.h b/src/iceberg/deletes/roaring_position_bitmap.h
index 73877d39..8d4b3586 100644
--- a/src/iceberg/deletes/roaring_position_bitmap.h
+++ b/src/iceberg/deletes/roaring_position_bitmap.h
@@ -28,7 +28,7 @@
#include <string>
#include <string_view>
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
namespace iceberg {
@@ -45,7 +45,7 @@ namespace iceberg {
/// \note This class is used to represent deletion vectors. The Puffin reader/writer
/// handle adding the additional required framing (length prefix, magic bytes, CRC-32)
/// for `deletion-vector-v1` persistence.
-class ICEBERG_EXPORT RoaringPositionBitmap {
+class ICEBERG_DATA_EXPORT RoaringPositionBitmap {
public:
/// \brief Maximum supported position (aligned with the Java implementation).
static constexpr int64_t kMaxPosition = 0x7FFFFFFE80000000LL;
diff --git a/src/iceberg/file_writer.h b/src/iceberg/file_writer.h
index 0b1af7f9..f3352d8f 100644
--- a/src/iceberg/file_writer.h
+++ b/src/iceberg/file_writer.h
@@ -36,7 +36,7 @@
namespace iceberg {
-class WriterProperties : public ConfigBase<WriterProperties> {
+class ICEBERG_EXPORT WriterProperties : public ConfigBase<WriterProperties> {
public:
template <typename T>
using Entry = const ConfigBase<WriterProperties>::Entry<T>;
diff --git a/src/iceberg/iceberg-config.cmake.in b/src/iceberg/iceberg-config.cmake.in
index 787fadcc..7a46cdd1 100644
--- a/src/iceberg/iceberg-config.cmake.in
+++ b/src/iceberg/iceberg-config.cmake.in
@@ -24,6 +24,8 @@
#
# iceberg::iceberg_shared
# iceberg::iceberg_static
+# iceberg::iceberg_data_shared
+# iceberg::iceberg_data_static
# iceberg::iceberg_bundle_shared
# iceberg::iceberg_bundle_static
# iceberg::iceberg_rest_shared
diff --git a/src/iceberg/arrow_c_data_guard_internal.h b/src/iceberg/iceberg_data_export.h
similarity index 50%
copy from src/iceberg/arrow_c_data_guard_internal.h
copy to src/iceberg/iceberg_data_export.h
index 8bce14e5..652de97b 100644
--- a/src/iceberg/arrow_c_data_guard_internal.h
+++ b/src/iceberg/iceberg_data_export.h
@@ -19,46 +19,16 @@
#pragma once
-#include <nanoarrow/nanoarrow.h>
-
-#include "iceberg/arrow_c_data.h"
-
-namespace iceberg::internal {
-
-class ArrowArrayGuard {
- public:
- explicit ArrowArrayGuard(ArrowArray* array) : array_(array) {}
- ~ArrowArrayGuard();
-
- private:
- ArrowArray* array_;
-};
-
-class ArrowSchemaGuard {
- public:
- explicit ArrowSchemaGuard(ArrowSchema* schema) : schema_(schema) {}
- ~ArrowSchemaGuard();
-
- private:
- ArrowSchema* schema_;
-};
-
-class ArrowArrayViewGuard {
- public:
- explicit ArrowArrayViewGuard(ArrowArrayView* view) : view_(view) {}
- ~ArrowArrayViewGuard();
-
- private:
- ArrowArrayView* view_;
-};
-
-class ArrowArrayBufferGuard {
- public:
- explicit ArrowArrayBufferGuard(ArrowBuffer* buffer) : buffer_(buffer) {}
- ~ArrowArrayBufferGuard();
-
- private:
- ArrowBuffer* buffer_;
-};
-
-} // namespace iceberg::internal
+#if defined(_WIN32) || defined(__CYGWIN__)
+# ifdef ICEBERG_DATA_STATIC
+# define ICEBERG_DATA_EXPORT
+# elif defined(ICEBERG_DATA_EXPORTING)
+# define ICEBERG_DATA_EXPORT __declspec(dllexport)
+# else
+# define ICEBERG_DATA_EXPORT __declspec(dllimport)
+# endif
+#else // Not Windows
+# ifndef ICEBERG_DATA_EXPORT
+# define ICEBERG_DATA_EXPORT __attribute__((visibility("default")))
+# endif
+#endif
diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build
index 5e68def9..17ed723f 100644
--- a/src/iceberg/meson.build
+++ b/src/iceberg/meson.build
@@ -42,14 +42,7 @@ iceberg_include_dir = include_directories('..')
iceberg_sources = files(
'arrow_c_data_guard_internal.cc',
'catalog/memory/in_memory_catalog.cc',
- 'data/data_writer.cc',
- 'data/delete_loader.cc',
- 'data/equality_delete_writer.cc',
- 'data/position_delete_writer.cc',
- 'data/writer.cc',
'delete_file_index.cc',
- 'deletes/position_delete_index.cc',
- 'deletes/roaring_position_bitmap.cc',
'expression/aggregate.cc',
'expression/binder.cc',
'expression/evaluator.cc',
@@ -88,9 +81,6 @@ iceberg_sources = files(
'partition_field.cc',
'partition_spec.cc',
'partition_summary.cc',
- 'puffin/file_metadata.cc',
- 'puffin/json_serde.cc',
- 'puffin/puffin_format.cc',
'row/arrow_array_wrapper.cc',
'row/manifest_wrapper.cc',
'row/partition_values.cc',
@@ -148,6 +138,19 @@ iceberg_sources = files(
'util/uuid.cc',
)
+iceberg_data_sources = files(
+ 'data/data_writer.cc',
+ 'data/delete_loader.cc',
+ 'data/equality_delete_writer.cc',
+ 'data/position_delete_writer.cc',
+ 'data/writer.cc',
+ 'deletes/position_delete_index.cc',
+ 'deletes/roaring_position_bitmap.cc',
+ 'puffin/file_metadata.cc',
+ 'puffin/json_serde.cc',
+ 'puffin/puffin_format.cc',
+)
+
# CRoaring does not export symbols, so on Windows it must
# be used as a static lib
croaring_needs_static = (
@@ -159,7 +162,7 @@ nanoarrow_dep = dependency('nanoarrow')
nlohmann_json_dep = dependency('nlohmann_json')
zlib_dep = dependency('zlib')
-iceberg_deps = [croaring_dep, nanoarrow_dep, nlohmann_json_dep, zlib_dep]
+iceberg_deps = [nanoarrow_dep, nlohmann_json_dep, zlib_dep]
iceberg_lib = library(
'iceberg',
@@ -184,8 +187,35 @@ iceberg_dep = declare_dependency(
compile_args: iceberg_interface_args,
)
meson.override_dependency('iceberg', iceberg_dep)
+
+iceberg_data_deps = [iceberg_dep, croaring_dep]
+iceberg_data_lib = library(
+ 'iceberg_data',
+ sources: iceberg_data_sources,
+ dependencies: iceberg_data_deps,
+ include_directories: iceberg_include_dir,
+ install: true,
+ gnu_symbol_visibility: 'inlineshidden',
+ cpp_shared_args: ['-DICEBERG_DATA_EXPORTING'],
+ cpp_static_args: ['-DICEBERG_DATA_STATIC'],
+)
+
+iceberg_data_interface_args = []
+if get_option('default_library') == 'static'
+ iceberg_data_interface_args += ['-DICEBERG_DATA_STATIC']
+endif
+
+iceberg_data_dep = declare_dependency(
+ link_with: iceberg_data_lib,
+ dependencies: iceberg_data_deps,
+ include_directories: iceberg_include_dir,
+ compile_args: iceberg_data_interface_args,
+)
+meson.override_dependency('iceberg-data', iceberg_data_dep)
+
pkg = import('pkgconfig')
pkg.generate(iceberg_lib)
+pkg.generate(iceberg_data_lib)
install_headers(
[
@@ -199,6 +229,7 @@ install_headers(
'file_io_registry.h',
'file_reader.h',
'file_writer.h',
+ 'iceberg_data_export.h',
'iceberg_export.h',
'inheritable_metadata.h',
'location_provider.h',
@@ -233,6 +264,7 @@ install_headers(
)
subdir('catalog')
+subdir('data')
subdir('deletes')
subdir('expression')
subdir('manifest')
diff --git a/src/iceberg/puffin/file_metadata.h b/src/iceberg/puffin/file_metadata.h
index 17ddad77..14eaae6c 100644
--- a/src/iceberg/puffin/file_metadata.h
+++ b/src/iceberg/puffin/file_metadata.h
@@ -29,7 +29,7 @@
#include <unordered_map>
#include <vector>
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
namespace iceberg::puffin {
@@ -41,12 +41,12 @@ enum class PuffinCompressionCodec {
kZstd,
};
-ICEBERG_EXPORT std::string_view CodecName(PuffinCompressionCodec codec);
+ICEBERG_DATA_EXPORT std::string_view CodecName(PuffinCompressionCodec codec);
-ICEBERG_EXPORT Result<PuffinCompressionCodec> PuffinCompressionCodecFromName(
+ICEBERG_DATA_EXPORT Result<PuffinCompressionCodec> PuffinCompressionCodecFromName(
std::string_view codec_name);
-ICEBERG_EXPORT std::string ToString(PuffinCompressionCodec codec);
+ICEBERG_DATA_EXPORT std::string ToString(PuffinCompressionCodec codec);
/// \brief Standard blob types defined by the Iceberg specification.
struct StandardBlobTypes {
@@ -67,7 +67,7 @@ struct StandardPuffinProperties {
};
/// \brief A blob in a Puffin file.
-struct ICEBERG_EXPORT Blob {
+struct ICEBERG_DATA_EXPORT Blob {
/// See StandardBlobTypes for known types.
std::string type;
/// Ordered list of field IDs the blob was computed from.
@@ -84,10 +84,10 @@ struct ICEBERG_EXPORT Blob {
friend bool operator==(const Blob& lhs, const Blob& rhs) = default;
};
-ICEBERG_EXPORT std::string ToString(const Blob& blob);
+ICEBERG_DATA_EXPORT std::string ToString(const Blob& blob);
/// \brief Metadata about a blob stored in a Puffin file footer.
-struct ICEBERG_EXPORT BlobMetadata {
+struct ICEBERG_DATA_EXPORT BlobMetadata {
/// See StandardBlobTypes for known types.
std::string type;
/// Ordered list of field IDs the blob was computed from.
@@ -105,16 +105,16 @@ struct ICEBERG_EXPORT BlobMetadata {
friend bool operator==(const BlobMetadata& lhs, const BlobMetadata& rhs) = default;
};
-ICEBERG_EXPORT std::string ToString(const BlobMetadata& blob_metadata);
+ICEBERG_DATA_EXPORT std::string ToString(const BlobMetadata& blob_metadata);
/// \brief Metadata about a Puffin file.
-struct ICEBERG_EXPORT FileMetadata {
+struct ICEBERG_DATA_EXPORT FileMetadata {
std::vector<BlobMetadata> blobs;
std::unordered_map<std::string, std::string> properties;
friend bool operator==(const FileMetadata& lhs, const FileMetadata& rhs) = default;
};
-ICEBERG_EXPORT std::string ToString(const FileMetadata& file_metadata);
+ICEBERG_DATA_EXPORT std::string ToString(const FileMetadata& file_metadata);
} // namespace iceberg::puffin
diff --git a/src/iceberg/puffin/json_serde_internal.h b/src/iceberg/puffin/json_serde_internal.h
index dbb5fbd0..9e003e16 100644
--- a/src/iceberg/puffin/json_serde_internal.h
+++ b/src/iceberg/puffin/json_serde_internal.h
@@ -27,30 +27,30 @@
#include <nlohmann/json_fwd.hpp>
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/puffin/type_fwd.h"
#include "iceberg/result.h"
namespace iceberg::puffin {
/// \brief Serialize a BlobMetadata to JSON.
-ICEBERG_EXPORT nlohmann::json ToJson(const BlobMetadata& blob_metadata);
+ICEBERG_DATA_EXPORT nlohmann::json ToJson(const BlobMetadata& blob_metadata);
/// \brief Deserialize a BlobMetadata from JSON.
-ICEBERG_EXPORT Result<BlobMetadata> BlobMetadataFromJson(const nlohmann::json& json);
+ICEBERG_DATA_EXPORT Result<BlobMetadata> BlobMetadataFromJson(const nlohmann::json& json);
/// \brief Serialize a FileMetadata to JSON.
-ICEBERG_EXPORT nlohmann::json ToJson(const FileMetadata& file_metadata);
+ICEBERG_DATA_EXPORT nlohmann::json ToJson(const FileMetadata& file_metadata);
/// \brief Deserialize a FileMetadata from JSON.
-ICEBERG_EXPORT Result<FileMetadata> FileMetadataFromJson(const nlohmann::json& json);
+ICEBERG_DATA_EXPORT Result<FileMetadata> FileMetadataFromJson(const nlohmann::json& json);
/// \brief Serialize a FileMetadata to a JSON string.
-ICEBERG_EXPORT std::string ToJsonString(const FileMetadata& file_metadata,
- bool pretty = false);
+ICEBERG_DATA_EXPORT std::string ToJsonString(const FileMetadata& file_metadata,
+ bool pretty = false);
/// \brief Deserialize a FileMetadata from a JSON string.
-ICEBERG_EXPORT Result<FileMetadata> FileMetadataFromJsonString(
+ICEBERG_DATA_EXPORT Result<FileMetadata> FileMetadataFromJsonString(
std::string_view json_string);
} // namespace iceberg::puffin
diff --git a/src/iceberg/puffin/puffin_format.h b/src/iceberg/puffin/puffin_format.h
index 857c2ba5..e5ecf900 100644
--- a/src/iceberg/puffin/puffin_format.h
+++ b/src/iceberg/puffin/puffin_format.h
@@ -26,14 +26,14 @@
#include <cstdint>
#include <span>
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
#include "iceberg/puffin/file_metadata.h"
#include "iceberg/result.h"
namespace iceberg::puffin {
/// \brief Puffin file format constants.
-struct ICEBERG_EXPORT PuffinFormat {
+struct ICEBERG_DATA_EXPORT PuffinFormat {
/// Magic bytes: "PFA1" (Puffin Fratercula arctica, version 1)
static constexpr std::array<uint8_t, 4> kMagicV1 = {0x50, 0x46, 0x41, 0x31};
@@ -61,9 +61,9 @@ enum class PuffinFlag : uint8_t {
};
/// \brief Check if a flag is set in the flags bytes.
-ICEBERG_EXPORT bool IsFlagSet(std::span<const uint8_t, 4> flags, PuffinFlag flag);
+ICEBERG_DATA_EXPORT bool IsFlagSet(std::span<const uint8_t, 4> flags, PuffinFlag flag);
/// \brief Set a flag in the flags bytes.
-ICEBERG_EXPORT void SetFlag(std::span<uint8_t, 4> flags, PuffinFlag flag);
+ICEBERG_DATA_EXPORT void SetFlag(std::span<uint8_t, 4> flags, PuffinFlag flag);
} // namespace iceberg::puffin
diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt
index f39b4e4f..3355dacf 100644
--- a/src/iceberg/test/CMakeLists.txt
+++ b/src/iceberg/test/CMakeLists.txt
@@ -31,7 +31,7 @@ set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources")
configure_file("test_config.h.in" "test_config.h")
function(add_iceberg_test test_name)
- set(options USE_BUNDLE)
+ set(options USE_BUNDLE USE_DATA)
set(oneValueArgs)
set(multiValueArgs SOURCES)
cmake_parse_arguments(ARG
@@ -47,6 +47,10 @@ function(add_iceberg_test test_name)
if(ARG_USE_BUNDLE)
target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gmock_main)
+ elseif(ARG_USE_DATA)
+ target_link_libraries(${test_name}
+ PRIVATE "$<IF:$<TARGET_EXISTS:iceberg_data_static>,iceberg_data_static,iceberg_data_shared>"
+ GTest::gmock_main)
else()
target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gmock_main)
endif()
@@ -109,6 +113,7 @@ add_iceberg_test(json_serde_test
schema_json_test.cc)
add_iceberg_test(util_test
+ USE_DATA
SOURCES
bucket_util_test.cc
config_test.cc
@@ -129,9 +134,11 @@ add_iceberg_test(util_test
uuid_test.cc
visit_type_test.cc)
-add_iceberg_test(roaring_test SOURCES roaring_test.cc)
-
-add_iceberg_test(puffin_test SOURCES puffin_format_test.cc puffin_json_test.cc)
+add_iceberg_test(puffin_test
+ USE_DATA
+ SOURCES
+ puffin_format_test.cc
+ puffin_json_test.cc)
if(ICEBERG_BUILD_BUNDLE)
add_iceberg_test(avro_test
diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build
index 3e88fc7e..e168d08b 100644
--- a/src/iceberg/test/meson.build
+++ b/src/iceberg/test/meson.build
@@ -102,10 +102,11 @@ iceberg_tests = {
'uuid_test.cc',
'visit_type_test.cc',
),
+ 'use_data': true,
},
- 'roaring_test': {'sources': files('roaring_test.cc')},
'puffin_test': {
'sources': files('puffin_format_test.cc', 'puffin_json_test.cc'),
+ 'use_data': true,
},
}
@@ -141,13 +142,17 @@ if get_option('rest').enabled()
endif
foreach test_name, values : iceberg_tests
+ test_deps = [gmock_main_dep] + values.get('dependencies', [])
+ if values.get('use_data', false)
+ test_deps = [iceberg_data_dep] + test_deps
+ else
+ test_deps = [iceberg_dep] + test_deps
+ endif
+
exc = executable(
test_name,
sources: values['sources'],
- dependencies: [iceberg_dep, gmock_main_dep] + values.get(
- 'dependencies',
- [],
- ),
+ dependencies: test_deps,
)
test(test_name, exc)
endforeach
diff --git a/src/iceberg/test/roaring_test.cc b/src/iceberg/test/roaring_test.cc
deleted file mode 100644
index 3c9e4d7a..00000000
--- a/src/iceberg/test/roaring_test.cc
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#include "roaring/roaring.hh"
-
-#include <gtest/gtest.h>
-
-#include "roaring/roaring64map.hh"
-
-namespace iceberg {
-
-TEST(CRoaringTest, Basic32Bit) {
- roaring::Roaring r1;
- for (uint32_t i = 100; i < 1000; i++) {
- r1.add(i);
- }
- ASSERT_EQ(r1.cardinality(), 900);
- ASSERT_TRUE(r1.contains(500));
- ASSERT_FALSE(r1.contains(50));
- ASSERT_FALSE(r1.isEmpty());
-}
-
-TEST(CRoaringTest, Basic64Bit) {
- roaring::Roaring64Map r2;
- for (uint64_t i = 18000000000000000100ull; i < 18000000000000001000ull; i++) {
- r2.add(i);
- }
- ASSERT_EQ(r2.cardinality(), 900);
- ASSERT_TRUE(r2.contains(static_cast<uint64_t>(18000000000000000500ull)));
- ASSERT_FALSE(r2.contains(static_cast<uint64_t>(18000000000000000050ull)));
- ASSERT_FALSE(r2.isEmpty());
-}
-
-TEST(CRoaringTest, ConstructorWithInitializerList) {
- roaring::Roaring r1 = roaring::Roaring::bitmapOf(5, 1, 2, 3, 5, 6);
- ASSERT_EQ(r1.cardinality(), 5);
- ASSERT_TRUE(r1.contains(1));
- ASSERT_TRUE(r1.contains(2));
- ASSERT_TRUE(r1.contains(3));
- ASSERT_TRUE(r1.contains(5));
- ASSERT_TRUE(r1.contains(6));
- ASSERT_FALSE(r1.contains(4));
-}
-
-} // namespace iceberg