You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2024/10/11 07:29:59 UTC
(iceberg-rust) branch gh-pages updated: deploy: 5c1a9e68da346819072a15327080a498ad91c488
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/gh-pages by this push:
new fed9c309 deploy: 5c1a9e68da346819072a15327080a498ad91c488
fed9c309 is described below
commit fed9c3097ea32f37ac69215fb3fd789cc522c6c3
Author: Xuanwo <Xu...@users.noreply.github.com>
AuthorDate: Fri Oct 11 07:29:54 2024 +0000
deploy: 5c1a9e68da346819072a15327080a498ad91c488
---
api/iceberg/arrow/fn.arrow_schema_to_schema.html | 2 +-
api/iceberg/arrow/index.html | 4 +-
api/iceberg/arrow/struct.ArrowReader.html | 6 +-
api/iceberg/arrow/struct.ArrowReaderBuilder.html | 12 +-
api/iceberg/enum.ErrorKind.html | 2 +-
api/iceberg/expr/enum.BoundPredicate.html | 4 +-
api/iceberg/expr/struct.BoundReference.html | 4 +-
api/iceberg/expr/struct.Reference.html | 4 +-
api/iceberg/expr/struct.SetExpression.html | 4 +-
api/iceberg/expr/struct.UnaryExpression.html | 4 +-
api/iceberg/index.html | 2 +-
api/iceberg/macro.ensure_data_valid.html | 2 +-
api/iceberg/scan/index.html | 2 +-
api/iceberg/scan/struct.FileScanTask.html | 9 +-
api/iceberg/spec/enum.FormatVersion.html | 4 +-
api/iceberg/spec/enum.NullOrder.html | 4 +-
api/iceberg/spec/enum.PrimitiveType.html | 4 +-
api/iceberg/spec/enum.SortDirection.html | 4 +-
api/iceberg/spec/enum.SortOrderBuilderError.html | 2 +-
api/iceberg/spec/enum.Type.html | 2 +-
api/iceberg/spec/struct.Datum.html | 2 +-
api/iceberg/spec/struct.Schema.html | 6 +-
api/iceberg/struct.Error.html | 6 +-
api/search-index.js | 2 +-
api/search.desc/iceberg/iceberg-desc-0-.js | 2 +-
api/src-files.js | 2 +-
api/src/iceberg/arrow/mod.rs.html | 4 +
api/src/iceberg/arrow/reader.rs.html | 20 +-
.../iceberg/arrow/record_batch_transformer.rs.html | 1245 ++++++++++++++++++++
api/src/iceberg/arrow/schema.rs.html | 2 +-
api/src/iceberg/error.rs.html | 12 +
api/src/iceberg/lib.rs.html | 2 +
api/src/iceberg/scan.rs.html | 54 +
api/trait.impl/core/convert/trait.From.js | 2 +-
34 files changed, 1391 insertions(+), 51 deletions(-)
diff --git a/api/iceberg/arrow/fn.arrow_schema_to_schema.html b/api/iceberg/arrow/fn.arrow_schema_to_schema.html
index b80b22ae..527f3f3b 100644
--- a/api/iceberg/arrow/fn.arrow_schema_to_schema.html
+++ b/api/iceberg/arrow/fn.arrow_schema_to_schema.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Convert Arrow schema to ceberg schema."><title>arrow_schema_to_schema in iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSa [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Convert Arrow schema to Iceberg schema."><title>arrow_schema_to_schema in iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraS [...]
</div></details></section></div></main></body></html>
\ No newline at end of file
diff --git a/api/iceberg/arrow/index.html b/api/iceberg/arrow/index.html
index aa57b5fb..09cde6ff 100644
--- a/api/iceberg/arrow/index.html
+++ b/api/iceberg/arrow/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Conversion between Iceberg and Arrow schema"><title>iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
-</div></details><h2 id="structs" class="section-header">Structs<a href="#structs" class="anchor">§</a></h2><ul class="item-table"><li><div class="item-name"><a class="struct" href="struct.ArrowReader.html" title="struct iceberg::arrow::ArrowReader">ArrowReader</a></div><div class="desc docblock-short">Reads data from Parquet files</div></li><li><div class="item-name"><a class="struct" href="struct.ArrowReaderBuilder.html" title="struct iceberg::arrow::ArrowReaderBuilder">ArrowReaderBuild [...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Conversion between Iceberg and Arrow schema"><title>iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
+</div></details><h2 id="structs" class="section-header">Structs<a href="#structs" class="anchor">§</a></h2><ul class="item-table"><li><div class="item-name"><a class="struct" href="struct.ArrowReader.html" title="struct iceberg::arrow::ArrowReader">ArrowReader</a></div><div class="desc docblock-short">Reads data from Parquet files</div></li><li><div class="item-name"><a class="struct" href="struct.ArrowReaderBuilder.html" title="struct iceberg::arrow::ArrowReaderBuilder">ArrowReaderBuild [...]
\ No newline at end of file
diff --git a/api/iceberg/arrow/struct.ArrowReader.html b/api/iceberg/arrow/struct.ArrowReader.html
index c02e2d3c..3c47abbe 100644
--- a/api/iceberg/arrow/struct.ArrowReader.html
+++ b/api/iceberg/arrow/struct.ArrowReader.html
@@ -1,7 +1,7 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Reads data from Parquet files"><title>ArrowReader in iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e49 [...]
-</div></details><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ArrowReader" class="impl"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#127-469">source</a><a href="#impl-ArrowReader" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.ArrowReader.html" title="struct [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Reads data from Parquet files"><title>ArrowReader in iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e49 [...]
+</div></details><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ArrowReader" class="impl"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#128-478">source</a><a href="#impl-ArrowReader" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.ArrowReader.html" title="struct [...]
Returns a stream of Arrow RecordBatches containing the data from the files</p>
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-ArrowReader" class="impl"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#115">source</a><a href="#impl-Clone-for-ArrowReader" class="anchor">§</a><h3 class="code-header">impl [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-ArrowReader" class="impl"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#116">source</a><a href="#impl-Clone-for-ArrowReader" class="anchor">§</a><h3 class="code-header">impl [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow_mut" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217">source</a><a href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a href= [...]
diff --git a/api/iceberg/arrow/struct.ArrowReaderBuilder.html b/api/iceberg/arrow/struct.ArrowReaderBuilder.html
index ce41b53b..11498724 100644
--- a/api/iceberg/arrow/struct.ArrowReaderBuilder.html
+++ b/api/iceberg/arrow/struct.ArrowReaderBuilder.html
@@ -1,13 +1,13 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Builder to create ArrowReader"><title>ArrowReaderBuilder in iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9 [...]
-</div></details><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ArrowReaderBuilder" class="impl"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#63-112">source</a><a href="#impl-ArrowReaderBuilder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.ArrowReaderBuilder [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_batch_size" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#85-88">source</a><h4 class="code-header">pub fn <a href="#method.with_batch_size" class="fn">with_batch_size</a>(self, batch_size: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>) -> Self</h4></section></summary><div class="docblock"><p>Sets the desi [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Builder to create ArrowReader"><title>ArrowReaderBuilder in iceberg::arrow - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9 [...]
+</div></details><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ArrowReaderBuilder" class="impl"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#64-113">source</a><a href="#impl-ArrowReaderBuilder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.ArrowReaderBuilder [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_batch_size" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#86-89">source</a><h4 class="code-header">pub fn <a href="#method.with_batch_size" class="fn">with_batch_size</a>(self, batch_size: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>) -> Self</h4></section></summary><div class="docblock"><p>Sets the desi [...]
to something other than the default</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_row_group_filtering_enabled" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#91-94">source</a><h4 class="code-header">pub fn <a href="#method.with_row_group_filtering_enabled" class="fn">with_row_group_filtering_enabled</a>(
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_row_group_filtering_enabled" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#92-95">source</a><h4 class="code-header">pub fn <a href="#method.with_row_group_filtering_enabled" class="fn">with_row_group_filtering_enabled</a>(
self,
row_group_filtering_enabled: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>,
) -> Self</h4></section></summary><div class="docblock"><p>Determines whether to enable row group filtering.</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_row_selection_enabled" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#97-100">source</a><h4 class="code-header">pub fn <a href="#method.with_row_selection_enabled" class="fn">with_row_selection_enabled</a>(self, row_selection_enabled: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>) -> Self</h4></section></sum [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.build" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#103-111">source</a><h4 class="code-header">pub fn <a href="#method.build" class="fn">build</a>(self) -> <a class="struct" href="struct.ArrowReader.html" title="struct iceberg::arrow::ArrowReader">ArrowReader</a></h4></section></summary><div class="docblock"><p>Build the ArrowReader.</p>
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_row_selection_enabled" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#98-101">source</a><h4 class="code-header">pub fn <a href="#method.with_row_selection_enabled" class="fn">with_row_selection_enabled</a>(self, row_selection_enabled: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>) -> Self</h4></section></sum [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.build" class="method"><a class="src rightside" href="../../src/iceberg/arrow/reader.rs.html#104-112">source</a><h4 class="code-header">pub fn <a href="#method.build" class="fn">build</a>(self) -> <a class="struct" href="struct.ArrowReader.html" title="struct iceberg::arrow::ArrowReader">ArrowReader</a></h4></section></summary><div class="docblock"><p>Build the ArrowReader.</p>
</div></details></div></details></div><h2 id="synthetic-implementations" class="section-header">Auto Trait Implementations<a href="#synthetic-implementations" class="anchor">§</a></h2><div id="synthetic-implementations-list"><section id="impl-Freeze-for-ArrowReaderBuilder" class="impl"><a href="#impl-Freeze-for-ArrowReaderBuilder" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Freeze.html" title="trait core::mar [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
diff --git a/api/iceberg/enum.ErrorKind.html b/api/iceberg/enum.ErrorKind.html
index 2e8a6b0c..7ffee91a 100644
--- a/api/iceberg/enum.ErrorKind.html
+++ b/api/iceberg/enum.ErrorKind.html
@@ -13,7 +13,7 @@ failed to parse it’s metadata or data file correctly.</p>
</div><section id="variant.FeatureUnsupported" class="variant"><a href="#variant.FeatureUnsupported" class="anchor">§</a><h3 class="code-header">FeatureUnsupported</h3></section><div class="docblock"><p>Iceberg feature is not supported.</p>
<p>This error is returned when given iceberg feature is not supported.</p>
</div></div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ErrorKind" class="impl"><a class="src rightside" href="../src/iceberg/error.rs.html#49-54">source</a><a href="#impl-ErrorKind" class="anchor">§</a><h3 class="code-header">impl <a class="enum" href="enum.ErrorKind.html" title="enum iceberg::ErrorKind">ErrorKi [...]
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-ErrorKind" class="impl"><a class="src rightside" href="../src/iceberg/error.rs.html#28">source</a><a href="#impl-Clone-for-ErrorKind" class="anchor">§</a><h3 class="code-header">impl <a class="trait [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-ErrorKind" class="impl"><a class="src rightside" href="../src/iceberg/error.rs.html#28">source</a><a href="#impl-Clone-for-ErrorKind" class="anchor">§</a><h3 class="code-header">impl <a class="trait [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><section id="impl-Copy-for-ErrorKind" class="impl"><a class="src rightside" href="../src/iceberg/error.rs.html#28">source</a><a href="#impl-Copy-for-ErrorKind" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Copy.html" title="trait core::marker::Copy">Copy</a> for <a class="enum" href="enum.ErrorKind.html" title="enu [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
diff --git a/api/iceberg/expr/enum.BoundPredicate.html b/api/iceberg/expr/enum.BoundPredicate.html
index 2c7416cd..58b05bd9 100644
--- a/api/iceberg/expr/enum.BoundPredicate.html
+++ b/api/iceberg/expr/enum.BoundPredicate.html
@@ -16,8 +16,8 @@
</div><section id="variant.Unary" class="variant"><a href="#variant.Unary" class="anchor">§</a><h3 class="code-header">Unary(<a class="struct" href="struct.UnaryExpression.html" title="struct iceberg::expr::UnaryExpression">UnaryExpression</a><<a class="struct" href="struct.BoundReference.html" title="struct iceberg::expr::BoundReference">BoundReference</a>>)</h3></section><div class="docblock"><p>Unary expression, for example, <code>a IS NULL</code>.</p>
</div><section id="variant.Binary" class="variant"><a href="#variant.Binary" class="anchor">§</a><h3 class="code-header">Binary(<a class="struct" href="struct.BinaryExpression.html" title="struct iceberg::expr::BinaryExpression">BinaryExpression</a><<a class="struct" href="struct.BoundReference.html" title="struct iceberg::expr::BoundReference">BoundReference</a>>)</h3></section><div class="docblock"><p>Binary expression, for example, <code>a > 10</code>.</p>
</div><section id="variant.Set" class="variant"><a href="#variant.Set" class="anchor">§</a><h3 class="code-header">Set(<a class="struct" href="struct.SetExpression.html" title="struct iceberg::expr::SetExpression">SetExpression</a><<a class="struct" href="struct.BoundReference.html" title="struct iceberg::expr::BoundReference">BoundReference</a>>)</h3></section><div class="docblock"><p>Set predicates, for example, <code>a IN (1, 2, 3)</code>.</p>
-</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-BoundPredicate" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#653">source</a><a href="#impl-Clone-for-BoundPredicate" class="anchor">§</a><h3 class="code-header">impl <a class="trait" h [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-BoundPredicate" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#653">source</a><a href="#impl-Clone-for-BoundPredicate" class="anchor">§</a><h3 class="code-header">impl <a class="trait" h [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-BoundPredicate" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#653">source</a><a href="#impl-Serialize-for-BoundPredicate" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="tr [...]
__S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><section id="impl-StructuralPartialEq-for-BoundPredicate" class="impl"><a class="src rightside" href=". [...]
diff --git a/api/iceberg/expr/struct.BoundReference.html b/api/iceberg/expr/struct.BoundReference.html
index 97ee66f3..d739d829 100644
--- a/api/iceberg/expr/struct.BoundReference.html
+++ b/api/iceberg/expr/struct.BoundReference.html
@@ -6,8 +6,8 @@
) -> Self</h4></section></summary><div class="docblock"><p>Creates a new bound reference.</p>
</div></details><details class="toggle method-toggle" open><summary><section id="method.field" class="method"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#366-368">source</a><h4 class="code-header">pub fn <a href="#method.field" class="fn">field</a>(&self) -> &<a class="struct" href="../spec/struct.NestedField.html" title="struct iceberg::spec::NestedField">NestedField</a></h4></section></summary><div class="docblock"><p>Return the field of this reference.</p>
</div></details><details class="toggle method-toggle" open><summary><section id="method.accessor" class="method"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#371-373">source</a><h4 class="code-header">pub fn <a href="#method.accessor" class="fn">accessor</a>(&self) -> &StructAccessor</h4></section></summary><div class="docblock"><p>Get this BoundReference’s Accessor</p>
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-BoundReference" class="impl"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#342">source</a><a href="#impl-Clone-for-BoundReference" class="anchor">§</a><h3 class="code-header">im [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-BoundReference" class="impl"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#342">source</a><a href="#impl-Clone-for-BoundReference" class="anchor">§</a><h3 class="code-header">im [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-BoundReference" class="impl"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#342">source</a><a href="#impl-Serialize-for-BoundReference" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait s [...]
__S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><section id="impl-Eq-for-BoundReference" class="impl"><a class="src rightside" href="../../src/iceberg/ [...]
diff --git a/api/iceberg/expr/struct.Reference.html b/api/iceberg/expr/struct.Reference.html
index 70a77fe6..d8ece61e 100644
--- a/api/iceberg/expr/struct.Reference.html
+++ b/api/iceberg/expr/struct.Reference.html
@@ -105,8 +105,8 @@ For example, <code>a</code> in <code>a > 10</code>.</p>
<span class="kw">let </span>as_string = <span class="macro">format!</span>(<span class="string">"{expr}"</span>);
<span class="macro">assert!</span>(<span class="kw-2">&</span>as_string == <span class="string">"a NOT IN (5, 6)" </span>|| <span class="kw-2">&</span>as_string == <span class="string">"a NOT IN (6, 5)"</span>);</code></pre></div>
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Bind-for-Reference" class="impl"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#309-339">source</a><a href="#impl-Bind-for-Reference" class="anchor">§</a><h3 class="code-header">impl <a cl [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Bind-for-Reference" class="impl"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#309-339">source</a><a href="#impl-Bind-for-Reference" class="anchor">§</a><h3 class="code-header">impl <a cl [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-Reference" class="impl"><a class="src rightside" href="../../src/iceberg/expr/term.rs.html#37">source</a><a href="#impl-Serialize-for-Reference" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serde::ser:: [...]
__S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><section id="impl-StructuralPartialEq-for-Reference" class="impl"><a class="src rightside" href="../../ [...]
diff --git a/api/iceberg/expr/struct.SetExpression.html b/api/iceberg/expr/struct.SetExpression.html
index 7d6e6dd5..3672862f 100644
--- a/api/iceberg/expr/struct.SetExpression.html
+++ b/api/iceberg/expr/struct.SetExpression.html
@@ -1,7 +1,7 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Set predicates, for example, `a in (1, 2, 3)`."><title>SetExpression in iceberg::expr - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans [...]
-</div></details><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Bind-for-SetExpression%3CT%3E" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#255-266">source</a><a href="#impl-Bind-for-SetExpression%3CT%3E" class="anchor">§</a><h3 class="code-header">impl& [...]
+</div></details><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Bind-for-SetExpression%3CT%3E" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#255-266">source</a><a href="#impl-Bind-for-SetExpression%3CT%3E" class="anchor">§</a><h3 class="code-header">impl& [...]
T: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html" title="trait serde::de::Deserialize">Deserialize</a><'de>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.deserialize" class="method trait-impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#214">source</a><a href="#method.deserialize" class="anchor">§</a><h4 class="code-header">fn <a hr [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-SetExpression%3CT%3E" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#214">source</a><a href="#impl-Serialize-for-SetExpression%3CT%3E" class="anchor">§</a><h3 class="code-header">impl<T> <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Seri [...]
T: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serde::ser::Serialize">Serialize</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.serialize" class="method trait-impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#214">source</a><a href="#method.serialize" class="anchor">§</a><h4 class="code-header">fn <a href="https://docs.rs [...]
diff --git a/api/iceberg/expr/struct.UnaryExpression.html b/api/iceberg/expr/struct.UnaryExpression.html
index e9faf0f7..fb84b98a 100644
--- a/api/iceberg/expr/struct.UnaryExpression.html
+++ b/api/iceberg/expr/struct.UnaryExpression.html
@@ -1,7 +1,7 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Unary predicate, for example, `a IS NULL`."><title>UnaryExpression in iceberg::expr - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-M [...]
-</div></details><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Bind-for-UnaryExpression%3CT%3E" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#125-132">source</a><a href="#impl-Bind-for-UnaryExpression%3CT%3E" class="anchor">§</a><h3 class="code-header">i [...]
+</div></details><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Bind-for-UnaryExpression%3CT%3E" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#125-132">source</a><a href="#impl-Bind-for-UnaryExpression%3CT%3E" class="anchor">§</a><h3 class="code-header">i [...]
T: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html" title="trait serde::de::Deserialize">Deserialize</a><'de>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.deserialize" class="method trait-impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#101">source</a><a href="#method.deserialize" class="anchor">§</a><h4 class="code-header">fn <a hr [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-UnaryExpression%3CT%3E" class="impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#101">source</a><a href="#impl-Serialize-for-UnaryExpression%3CT%3E" class="anchor">§</a><h3 class="code-header">impl<T> <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait. [...]
T: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serde::ser::Serialize">Serialize</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.serialize" class="method trait-impl"><a class="src rightside" href="../../src/iceberg/expr/predicate.rs.html#101">source</a><a href="#method.serialize" class="anchor">§</a><h4 class="code-header">fn <a href="https://docs.rs [...]
diff --git a/api/iceberg/index.html b/api/iceberg/index.html
index 0815427c..b77a56f0 100644
--- a/api/iceberg/index.html
+++ b/api/iceberg/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Apache Iceberg Official Native Rust Implementation"><title>iceberg - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Apache Iceberg Official Native Rust Implementation"><title>iceberg - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
<h2 id="examples"><a class="doc-anchor" href="#examples">§</a>Examples</h2><h3 id="scan-a-table"><a class="doc-anchor" href="#scan-a-table">§</a>Scan A Table</h3>
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="kw">use </span>futures::TryStreamExt;
<span class="kw">use </span>iceberg::io::{FileIO, FileIOBuilder};
diff --git a/api/iceberg/macro.ensure_data_valid.html b/api/iceberg/macro.ensure_data_valid.html
index 18a02f45..7ba10694 100644
--- a/api/iceberg/macro.ensure_data_valid.html
+++ b/api/iceberg/macro.ensure_data_valid.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Helper macro to check arguments."><title>ensure_data_valid in iceberg - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Helper macro to check arguments."><title>ensure_data_valid in iceberg - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e [...]
(<span class="macro-nonterminal">$cond</span>: expr, <span class="macro-nonterminal">$fmt</span>: literal, $(<span class="macro-nonterminal">$arg</span>:tt)<span class="kw-2">*</span>) => { ... };
}</pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>Helper macro to check arguments.</p>
<p>Example:</p>
diff --git a/api/iceberg/scan/index.html b/api/iceberg/scan/index.html
index 5a0b6bdf..44c0a1c5 100644
--- a/api/iceberg/scan/index.html
+++ b/api/iceberg/scan/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Table scan api."><title>iceberg::scan - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Reg [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Table scan api."><title>iceberg::scan - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Reg [...]
</div></details><h2 id="structs" class="section-header">Structs<a href="#structs" class="anchor">§</a></h2><ul class="item-table"><li><div class="item-name"><a class="struct" href="struct.FileScanTask.html" title="struct iceberg::scan::FileScanTask">FileScanTask</a></div><div class="desc docblock-short">A task to scan part of file.</div></li><li><div class="item-name"><a class="struct" href="struct.TableScan.html" title="struct iceberg::scan::TableScan">TableScan</a></div><div class="des [...]
\ No newline at end of file
diff --git a/api/iceberg/scan/struct.FileScanTask.html b/api/iceberg/scan/struct.FileScanTask.html
index b1e04c25..358fa3bc 100644
--- a/api/iceberg/scan/struct.FileScanTask.html
+++ b/api/iceberg/scan/struct.FileScanTask.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A task to scan part of file."><title>FileScanTask in iceberg::scan - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A task to scan part of file."><title>FileScanTask in iceberg::scan - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
pub start: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u64.html">u64</a>,
pub length: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u64.html">u64</a>,
pub record_count: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u64.html">u64</a>>,
@@ -20,7 +20,12 @@ reading the entire data file.</p>
</div><span id="structfield.schema" class="structfield section-header"><a href="#structfield.schema" class="anchor field">§</a><code>schema: <a class="type" href="../spec/type.SchemaRef.html" title="type iceberg::spec::SchemaRef">SchemaRef</a></code></span><div class="docblock"><p>The schema of the file to scan.</p>
</div><span id="structfield.project_field_ids" class="structfield section-header"><a href="#structfield.project_field_ids" class="anchor field">§</a><code>project_field_ids: <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.i32.html">i32</a>></code></span><div class="docblock"><p>The field ids to project.</p>
</div><span id="structfield.predicate" class="structfield section-header"><a href="#structfield.predicate" class="anchor field">§</a><code>predicate: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="enum" href="../expr/enum.BoundPredicate.html" title="enum iceberg::expr::BoundPredicate">BoundPredicate</a>></code></span><div class="docblock"><p>The predicate to filter.</p>
-</div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-FileScanTask" class="impl"><a class="src rightside" href="../../src/iceberg/scan.rs.html#881">source</a><a href="#impl-Clone-for-FileScanTask" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://doc.rus [...]
+</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-FileScanTask" class="impl"><a class="src rightside" href="../../src/iceberg/scan.rs.html#909-934">source</a><a href="#impl-FileScanTask" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.FileScanTask.html" title="struct iceberg::scan: [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.project_field_ids" class="method"><a class="src rightside" href="../../src/iceberg/scan.rs.html#916-918">source</a><h4 class="code-header">pub fn <a href="#method.project_field_ids" class="fn">project_field_ids</a>(&self) -> &[<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.i32.html">i32</a>]</h4></section></summary><div class="docblock"><p>Returns the project fie [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.predicate" class="method"><a class="src rightside" href="../../src/iceberg/scan.rs.html#921-923">source</a><h4 class="code-header">pub fn <a href="#method.predicate" class="fn">predicate</a>(&self) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><&<a class="enum" href="../expr/enum.BoundPredicate.html" t [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.schema" class="method"><a class="src rightside" href="../../src/iceberg/scan.rs.html#926-928">source</a><h4 class="code-header">pub fn <a href="#method.schema" class="fn">schema</a>(&self) -> &<a class="struct" href="../spec/struct.Schema.html" title="struct iceberg::spec::Schema">Schema</a></h4></section></summary><div class="docblock"><p>Returns the schema of this file scan task as a refe [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.schema_ref" class="method"><a class="src rightside" href="../../src/iceberg/scan.rs.html#931-933">source</a><h4 class="code-header">pub fn <a href="#method.schema_ref" class="fn">schema_ref</a>(&self) -> <a class="type" href="../spec/type.SchemaRef.html" title="type iceberg::spec::SchemaRef">SchemaRef</a></h4></section></summary><div class="docblock"><p>Returns the schema of this file scan task [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-FileScanTask" class="impl"><a class="src rightside" href="../../src/iceberg/scan.rs.html#881">source</a><a href="#impl-Clone-for-FileScanTask" class="anchor">§</a><h3 class="code-header">impl <a cla [...]
__D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Seria [...]
__S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details></div><h2 id="synthetic-implementations" class="section-header">Auto Trait Implementations<a href="#syn [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
diff --git a/api/iceberg/spec/enum.FormatVersion.html b/api/iceberg/spec/enum.FormatVersion.html
index d2fc7ae3..dd923e71 100644
--- a/api/iceberg/spec/enum.FormatVersion.html
+++ b/api/iceberg/spec/enum.FormatVersion.html
@@ -4,8 +4,8 @@
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>Iceberg format version</p>
</div></details><h2 id="variants" class="variants section-header">Variants<a href="#variants" class="anchor">§</a></h2><div class="variants"><section id="variant.V1" class="variant"><a href="#variant.V1" class="anchor">§</a><h3 class="code-header">V1 = 1</h3></section><div class="docblock"><p>Iceberg spec version 1</p>
</div><section id="variant.V2" class="variant"><a href="#variant.V2" class="anchor">§</a><h3 class="code-header">V2 = 2</h3></section><div class="docblock"><p>Iceberg spec version 2</p>
-</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-FormatVersion" class="impl"><a class="src rightside" href="../../src/iceberg/spec/table_metadata.rs.html#1121">source</a><a href="#impl-Clone-for-FormatVersion" class="anchor">§</a><h3 class="code-header">impl <a class="trai [...]
- D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Display [...]
+</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-FormatVersion" class="impl"><a class="src rightside" href="../../src/iceberg/spec/table_metadata.rs.html#1121">source</a><a href="#impl-Clone-for-FormatVersion" class="anchor">§</a><h3 class="code-header">impl <a class="trai [...]
+ D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Display [...]
H: <a class="trait" href="https://doc.rust-lang.org/nightly/core/hash/trait.Hasher.html" title="trait core::hash::Hasher">Hasher</a>,
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h4></section></summary><div class='docblock'>Feeds a slice of this type into the given <a href="https://doc.rust-lang.org/nightly/core/hash/trait.Hasher.html" title="trait core::hash::Hasher"><code>Hasher</code></a>. <a href="https://doc.rust-lang.org/nightly/core/hash/trait.Hash.html#method.hash_slice">Read more</a></div></details></div></d [...]
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h4></section></summary><div class='docblock'>Compares and returns the maximum of two values. <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.Ord.html#method.max">Read more</a></div></details><details class="toggle method-toggle" open><summary><section id="method.min" class="method trait-impl"><span class="rightside"><span class="si [...]
diff --git a/api/iceberg/spec/enum.NullOrder.html b/api/iceberg/spec/enum.NullOrder.html
index 302cda17..e43f63f1 100644
--- a/api/iceberg/spec/enum.NullOrder.html
+++ b/api/iceberg/spec/enum.NullOrder.html
@@ -4,8 +4,8 @@
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>Describes the order of null values when sorted.</p>
</div></details><h2 id="variants" class="variants section-header">Variants<a href="#variants" class="anchor">§</a></h2><div class="variants"><section id="variant.First" class="variant"><a href="#variant.First" class="anchor">§</a><h3 class="code-header">First</h3></section><div class="docblock"><p>Nulls are stored first</p>
</div><section id="variant.Last" class="variant"><a href="#variant.Last" class="anchor">§</a><h3 class="code-header">Last</h3></section><div class="docblock"><p>Nulls are stored last</p>
-</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-NullOrder" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#55">source</a><a href="#impl-Clone-for-NullOrder" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://doc [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-NullOrder" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#55">source</a><a href="#impl-Clone-for-NullOrder" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://doc [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-NullOrder" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#55">source</a><a href="#impl-Serialize-for-NullOrder" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serde::ser:: [...]
__S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><section id="impl-Copy-for-NullOrder" class="impl"><a class="src rightside" href="../../src/iceberg/spe [...]
diff --git a/api/iceberg/spec/enum.PrimitiveType.html b/api/iceberg/spec/enum.PrimitiveType.html
index bb03ba07..10e1d525 100644
--- a/api/iceberg/spec/enum.PrimitiveType.html
+++ b/api/iceberg/spec/enum.PrimitiveType.html
@@ -45,8 +45,8 @@
__deserializer: __D,
) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><<a class="enum" href="enum.PrimitiveType.html" title="enum iceberg::spec::PrimitiveType">PrimitiveType</a>, __D::<a class="associatedtype" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html#associatedtype.Error" title="type serde::de::Deserializer::Error">Error</a>><div class="where">where
__D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-PrimitiveType-2" class="impl"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#242-265">source</a><a href="#impl-PrimitiveType-2" class="anchor">§</a><h3 class="code-header">impl <a class="enum" [...]
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-PrimitiveType" class="impl"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#200">source</a><a href="#impl-Clone-for-PrimitiveType" class="anchor">§</a><h3 class="code-header" [...]
- D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Display [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-PrimitiveType" class="impl"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#200">source</a><a href="#impl-Clone-for-PrimitiveType" class="anchor">§</a><h3 class="code-header" [...]
+ D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Display [...]
H: <a class="trait" href="https://doc.rust-lang.org/nightly/core/hash/trait.Hasher.html" title="trait core::hash::Hasher">Hasher</a>,
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h4></section></summary><div class='docblock'>Feeds a slice of this type into the given <a href="https://doc.rust-lang.org/nightly/core/hash/trait.Hasher.html" title="trait core::hash::Hasher"><code>Hasher</code></a>. <a href="https://doc.rust-lang.org/nightly/core/hash/trait.Hash.html#method.hash_slice">Read more</a></div></details></div></d [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
diff --git a/api/iceberg/spec/enum.SortDirection.html b/api/iceberg/spec/enum.SortDirection.html
index c4963f97..424e379c 100644
--- a/api/iceberg/spec/enum.SortDirection.html
+++ b/api/iceberg/spec/enum.SortDirection.html
@@ -4,8 +4,8 @@
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>Sort direction in a partition, either ascending or descending</p>
</div></details><h2 id="variants" class="variants section-header">Variants<a href="#variants" class="anchor">§</a></h2><div class="variants"><section id="variant.Ascending" class="variant"><a href="#variant.Ascending" class="anchor">§</a><h3 class="code-header">Ascending</h3></section><div class="docblock"><p>Ascending</p>
</div><section id="variant.Descending" class="variant"><a href="#variant.Descending" class="anchor">§</a><h3 class="code-header">Descending</h3></section><div class="docblock"><p>Descending</p>
-</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-SortDirection" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#35">source</a><a href="#impl-Clone-for-SortDirection" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="htt [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-SortDirection" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#35">source</a><a href="#impl-Clone-for-SortDirection" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="htt [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-SortDirection" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#35">source</a><a href="#impl-Serialize-for-SortDirection" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serd [...]
__S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><section id="impl-Copy-for-SortDirection" class="impl"><a class="src rightside" href="../../src/iceberg [...]
diff --git a/api/iceberg/spec/enum.SortOrderBuilderError.html b/api/iceberg/spec/enum.SortOrderBuilderError.html
index 65d194bb..297efdae 100644
--- a/api/iceberg/spec/enum.SortOrderBuilderError.html
+++ b/api/iceberg/spec/enum.SortOrderBuilderError.html
@@ -4,7 +4,7 @@
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>Error type for SortOrderBuilder</p>
</div></details><h2 id="variants" class="variants section-header">Variants (Non-exhaustive)<a href="#variants" class="anchor">§</a></h2><details class="toggle non-exhaustive"><summary class="hideme"><span>This enum is marked as non-exhaustive</span></summary><div class="docblock">Non-exhaustive enums could have additional variants added in future. Therefore, when matching against variants of non-exhaustive enums, an extra wildcard arm must be added to account for any future variants.</di [...]
</div><section id="variant.ValidationError" class="variant"><a href="#variant.ValidationError" class="anchor">§</a><h3 class="code-header">ValidationError(<a class="struct" href="https://doc.rust-lang.org/nightly/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>)</h3></section><div class="docblock"><p>Custom validation error</p>
-</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-SortOrderBuilderError" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#99">source</a><a href="#impl-Debug-for-SortOrderBuilderError" class="anchor">§</a><h3 class="code-header">impl <a class=" [...]
+</div></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-SortOrderBuilderError" class="impl"><a class="src rightside" href="../../src/iceberg/spec/sort.rs.html#99">source</a><a href="#impl-Debug-for-SortOrderBuilderError" class="anchor">§</a><h3 class="code-header">impl <a class=" [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: <a class="trait" href="https://doc.rust-lang.org/nightly/core/error/trait.Error.html" title="trait core::error::Error">Error</a> + 'static,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.as_error_source" class="method trait-impl"><a href="#method.as_error_source" class="anchor">§</a><h4 class="code-header">fn <a class="fn">as_error_source</a>(&self) -> &(dyn <a class="trait" href="https://doc [...]
to benefit from Rust’s automatic dereferencing of method
diff --git a/api/iceberg/spec/enum.Type.html b/api/iceberg/spec/enum.Type.html
index c6f9e5fb..62401550 100644
--- a/api/iceberg/spec/enum.Type.html
+++ b/api/iceberg/spec/enum.Type.html
@@ -18,7 +18,7 @@
</div></details><details class="toggle method-toggle" open><summary><section id="method.decimal" class="method"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#160-163">source</a><h4 class="code-header">pub fn <a href="#method.decimal" class="fn">decimal</a>(precision: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u32.html">u32</a>, scale: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u32.html">u32</a>) -&g [...]
</div></details><details class="toggle method-toggle" open><summary><section id="method.is_floating_type" class="method"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#167-172">source</a><h4 class="code-header">pub fn <a href="#method.is_floating_type" class="fn">is_floating_type</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></h4></section></summary><div class="docblock"><p>Check if it’s float o [...]
</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-Type" class="impl"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#80">source</a><a href="#impl-Clone-for-Type" class="anchor">§</a><h3 class="code-header">impl <a class="tra [...]
- D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Display [...]
+ D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Display [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-Type" class="impl"><a class="src rightside" href="../../src/iceberg/spec/datatypes.rs.html#267-273">source</a><a href="#impl-Serialize-for-Type" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serde::ser:: [...]
S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><section id="impl-Eq-for-Type" class="impl"><a class="src rightside" href="../../src/iceberg/spec/datatyp [...]
diff --git a/api/iceberg/spec/struct.Datum.html b/api/iceberg/spec/struct.Datum.html
index 51de3d28..dc4d8f39 100644
--- a/api/iceberg/spec/struct.Datum.html
+++ b/api/iceberg/spec/struct.Datum.html
@@ -265,7 +265,7 @@ Example:</p>
</div></details><details class="toggle method-toggle" open><summary><section id="method.data_type" class="method"><a class="src rightside" href="../../src/iceberg/spec/values.rs.html#1049-1051">source</a><h4 class="code-header">pub fn <a href="#method.data_type" class="fn">data_type</a>(&self) -> &<a class="enum" href="enum.PrimitiveType.html" title="enum iceberg::spec::PrimitiveType">PrimitiveType</a></h4></section></summary><div class="docblock"><p>Get the primitive type fro [...]
</div></details><details class="toggle method-toggle" open><summary><section id="method.is_nan" class="method"><a class="src rightside" href="../../src/iceberg/spec/values.rs.html#1055-1061">source</a><h4 class="code-header">pub fn <a href="#method.is_nan" class="fn">is_nan</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></h4></section></summary><div class="docblock"><p>Returns true if the Literal represents a primitive type
that can be a NaN, and that it’s value is NaN</p>
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-Datum" class="impl"><a class="src rightside" href="../../src/iceberg/spec/values.rs.html#96">source</a><a href="#impl-Clone-for-Datum" class="anchor">§</a><h3 class="code-header">impl <a class="trai [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-Datum" class="impl"><a class="src rightside" href="../../src/iceberg/spec/values.rs.html#96">source</a><a href="#impl-Clone-for-Datum" class="anchor">§</a><h3 class="code-header">impl <a class="trai [...]
H: <a class="trait" href="https://doc.rust-lang.org/nightly/core/hash/trait.Hasher.html" title="trait core::hash::Hasher">Hasher</a>,
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h4></section></summary><div class='docblock'>Feeds a slice of this type into the given <a href="https://doc.rust-lang.org/nightly/core/hash/trait.Hasher.html" title="trait core::hash::Hasher"><code>Hasher</code></a>. <a href="https://doc.rust-lang.org/nightly/core/hash/trait.Hash.html#method.hash_slice">Read more</a></div></details></div></d [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
diff --git a/api/iceberg/spec/struct.Schema.html b/api/iceberg/spec/struct.Schema.html
index 5585e172..c35b4349 100644
--- a/api/iceberg/spec/struct.Schema.html
+++ b/api/iceberg/spec/struct.Schema.html
@@ -17,11 +17,11 @@
</div></details><details class="toggle method-toggle" open><summary><section id="method.field_id_by_name" class="method"><a class="src rightside" href="../../src/iceberg/spec/schema.rs.html#366-368">source</a><h4 class="code-header">pub fn <a href="#method.field_id_by_name" class="fn">field_id_by_name</a>(&self, name: &<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/op [...]
</div></details><details class="toggle method-toggle" open><summary><section id="method.name_by_field_id" class="method"><a class="src rightside" href="../../src/iceberg/spec/schema.rs.html#371-373">source</a><h4 class="code-header">pub fn <a href="#method.name_by_field_id" class="fn">name_by_field_id</a>(&self, field_id: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.i32.html">i32</a>) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/opt [...]
</div></details><details class="toggle method-toggle" open><summary><section id="method.accessor_by_field_id" class="method"><a class="src rightside" href="../../src/iceberg/spec/schema.rs.html#376-378">source</a><h4 class="code-header">pub fn <a href="#method.accessor_by_field_id" class="fn">accessor_by_field_id</a>(&self, field_id: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.i32.html">i32</a>) -> <a class="enum" href="https://doc.rust-lang.org/nigh [...]
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-Schema" class="impl"><a class="src rightside" href="../../src/iceberg/spec/schema.rs.html#45">source</a><a href="#impl-Clone-for-Schema" class="anchor">§</a><h3 class="code-header">impl <a class="tr [...]
- __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-Schema" class="impl"><a class="src rightside" href="../../src/iceberg/spec/schema.rs.html#45">source</a><a href="#impl-Clone-for-Schema" class="anchor">§</a><h3 class="code-header">impl <a class="tr [...]
+ __D: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserializer.html" title="trait serde::de::Deserializer">Deserializer</a><'de>,</div></h4></section></summary><div class='docblock'>Deserialize this value from the given Serde deserializer. <a href="https://docs.rs/serde/1.0.210/serde/de/trait.Deserialize.html#tymethod.deserialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Displ [...]
by <code>==</code>.</div></details><details class="toggle method-toggle" open><summary><section id="method.ne" class="method trait-impl"><span class="rightside"><span class="since" title="Stable since Rust version 1.0.0">1.0.0</span> · <a class="src" href="https://doc.rust-lang.org/nightly/src/core/cmp.rs.html#263">source</a></span><a href="#method.ne" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialEq.html#method.ne" class= [...]
sufficient, and should not be overridden without very good reason.</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-Serialize-for-Schema" class="impl"><a class="src rightside" href="../../src/iceberg/spec/schema.rs.html#45">source</a><a href="#impl-Serialize-for-Schema" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html" title="trait serde::ser::Seri [...]
- __S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-TryFrom%3C%26Schema%3E-for- [...]
+ __S: <a class="trait" href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serializer.html" title="trait serde::ser::Serializer">Serializer</a>,</div></h4></section></summary><div class='docblock'>Serialize this value into the given Serde serializer. <a href="https://docs.rs/serde/1.0.210/serde/ser/trait.Serialize.html#tymethod.serialize">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-TryFrom%3C%26Schema%3E-for- [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow_mut" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217">source</a><a href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a href= [...]
diff --git a/api/iceberg/struct.Error.html b/api/iceberg/struct.Error.html
index 46d5ea8c..5262370c 100644
--- a/api/iceberg/struct.Error.html
+++ b/api/iceberg/struct.Error.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Error is the error struct returned by all iceberg functions."><title>Error in iceberg - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Error is the error struct returned by all iceberg functions."><title>Error in iceberg - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans [...]
<h3 id="display"><a class="doc-anchor" href="#display">§</a>Display</h3>
<p>Error can be displayed in two ways:</p>
<ul>
@@ -35,7 +35,7 @@ Backtrace:
</div></details><details class="toggle method-toggle" open><summary><section id="method.kind" class="method"><a class="src rightside" href="../src/iceberg/error.rs.html#241-243">source</a><h4 class="code-header">pub fn <a href="#method.kind" class="fn">kind</a>(&self) -> <a class="enum" href="enum.ErrorKind.html" title="enum iceberg::ErrorKind">ErrorKind</a></h4></section></summary><div class="docblock"><p>Return error’s kind.</p>
<p>Users can use this method to check error’s kind and take actions.</p>
</div></details><details class="toggle method-toggle" open><summary><section id="method.message" class="method"><a class="src rightside" href="../src/iceberg/error.rs.html#247-249">source</a><h4 class="code-header">pub fn <a href="#method.message" class="fn">message</a>(&self) -> &<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a></h4></section></summary><div class="docblock"><p>Return error’s message.</p>
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-Error" class="impl"><a class="src rightside" href="../src/iceberg/error.rs.html#148-187">source</a><a href="#impl-Debug-for-Error" class="anchor">§</a><h3 class="code-header">impl <a class="trait" h [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-Error" class="impl"><a class="src rightside" href="../src/iceberg/error.rs.html#148-187">source</a><a href="#impl-Debug-for-Error" class="anchor">§</a><h3 class="code-header">impl <a class="trait" h [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: <a class="trait" href="https://doc.rust-lang.org/nightly/core/error/trait.Error.html" title="trait core::error::Error">Error</a> + 'static,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.as_error_source" class="method trait-impl"><a href="#method.as_error_source" class="anchor">§</a><h4 class="code-header">fn <a class="fn">as_error_source</a>(&self) -> &(dyn <a class="trait" href="https://doc [...]
to benefit from Rust’s automatic dereferencing of method
@@ -57,7 +57,7 @@ receivers.</div></details></div></details><details class="toggle implementors-to
<code>Debug</code>-formatted.</div></details><details class="toggle method-toggle" open><summary><section id="method.fmt_upper_hex" class="method trait-impl"><a href="#method.fmt_upper_hex" class="anchor">§</a><h4 class="code-header">fn <a class="fn">fmt_upper_hex</a>(self) -> FmtUpperHex<Self><div class="where">where
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/fmt/trait.UpperHex.html" title="trait core::fmt::UpperHex">UpperHex</a>,</div></h4></section></summary><div class='docblock'>Causes <code>self</code> to use its <code>UpperHex</code> implementation when
<code>Debug</code>-formatted.</div></details><details class="toggle method-toggle" open><summary><section id="method.fmt_list" class="method trait-impl"><a href="#method.fmt_list" class="anchor">§</a><h4 class="code-header">fn <a class="fn">fmt_list</a>(self) -> FmtList<Self><div class="where">where
- <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.reference.html">&'a Self</a>: for<'a> <a class="trait" href="https://doc.rust-lang.org/nightly/core/iter/traits/collect/trait.IntoIterator.html" title="trait core::iter::traits::collect::IntoIterator">IntoIterator</a>,</div></h4></section></summary><div class='docblock'>Formats each item in a sequence. <a>Read more</a></div></details></div></details><details class="toggle implementors-toggle"><summar [...]
+ <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.reference.html">&'a Self</a>: for<'a> <a class="trait" href="https://doc.rust-lang.org/nightly/core/iter/traits/collect/trait.IntoIterator.html" title="trait core::iter::traits::collect::IntoIterator">IntoIterator</a>,</div></h4></section></summary><div class='docblock'>Formats each item in a sequence. <a>Read more</a></div></details></div></details><details class="toggle implementors-toggle"><summar [...]
</div></details></div></details><details class="toggle implementors-toggle"><summary><section id="impl-Instrument-for-T" class="impl"><a href="#impl-Instrument-for-T" class="anchor">§</a><h3 class="code-header">impl<T> Instrument for T</h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.instrument" class="method trait-impl"><a href="#method.instrument" class="anchor">§</a><h4 class="code-header">fn <a class="fn">in [...]
<code>Instrumented</code> wrapper. <a>Read more</a></div></details><details class="toggle method-toggle" open><summary><section id="method.in_current_span" class="method trait-impl"><a href="#method.in_current_span" class="anchor">§</a><h4 class="code-header">fn <a class="fn">in_current_span</a>(self) -> Instrumented<Self></h4></section></summary><div class='docblock'>Instruments this type with the <a href="super::Span::current()">current</a> <a href="crate::Span"><code>Span</co [...]
<code>Instrumented</code> wrapper. <a>Read more</a></div></details></div></details><details class="toggle implementors-toggle"><summary><section id="impl-Into%3CU%3E-for-T" class="impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/convert/mod.rs.html#748-750">source</a><a href="#impl-Into%3CU%3E-for-T" class="anchor">§</a><h3 class="code-header">impl<T, U> <a class="trait" href="https://doc.rust-lang.org/nightly/core/convert/trait.Into.html" title="trai [...]
diff --git a/api/search-index.js b/api/search-index.js
index 29cdf121..35b4d460 100644
--- a/api/search-index.js
+++ b/api/search-index.js
@@ -1,5 +1,5 @@
var searchIndex = new Map(JSON.parse('[\
-["iceberg",{"t":"PPPPPKPPPPPFGPPPFFPPPPPPIPPPPPPFFFGGPPPFNCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMQNNNNNNNNNNNNNNCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMNNNNNNNNNNNNNNNNNNNNNNNNNCNMMMOONNNOOONOMNNNNNNONOOMOCOONNNNONCOCMNNNNNNNNNNCCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMNNNNNNNNNNNNNCOOOOOOOOOOOOOOOOOOOOOOOOOFFKRRNNNNNNNNHHNNNNNNNNNNNNNNNNNNNNNNNNNNNMMMNMHMNNNNNNNHNNNNNNPPPPPPPPFKRGFIPPPPPPPPFPPPPPPPPPGGFPPFPIPPFNNMNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN [...]
+["iceberg",{"t":"PPPPPKPPPPPFGPPPFFPPPPPPIPPPPPPFFFGGPPPFNCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMQNNNNNNNNNNNNNNCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMNNNNNNNNNNNNNNNNNNNNNNNNNCNMMMOONNNOOONOMNNNNNNONOOMOCOONNNNONCOCMNNNNNNNNNNCCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMNNNNNNNNNNNNNCOOOOOOOOOOOOOOOOOOOOOOOOOFFKRRNNNNNNNNHHNNNNNNNNNNNNNNNNNNNNNNNNNNNMMMNMHMNNNNNNNHNNNNNNPPPPPPPPFKRGFIPPPPPPPPFPPPPPPPPPGGFPPFPIPPFNNMNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN [...]
["iceberg_catalog_glue",{"t":"SSSSSFFNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNQ","n":["AWS_ACCESS_KEY_ID","AWS_PROFILE_NAME","AWS_REGION_NAME","AWS_SECRET_ACCESS_KEY","AWS_SESSION_TOKEN","GlueCatalog","GlueCatalogConfig","borrow","borrow","borrow_mut","borrow_mut","builder","create_namespace","create_table","deref","deref","deref_mut","deref_mut","drop","drop","drop_namespace","drop_table","file_io","fmt","fmt","from","from","get_namespace","init","init","into","into","into_shared","i [...]
["iceberg_catalog_hms",{"t":"PPFFGNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN","n":["Buffered","Framed","HmsCatalog","HmsCatalogConfig","HmsThriftTransport","borrow","borrow","borrow","borrow_mut","borrow_mut","borrow_mut","builder","create_namespace","create_table","default","deref","deref","deref","deref_mut","deref_mut","deref_mut","drop","drop","drop","drop_namespace","drop_table","file_io","fmt","fmt","fmt","from","from","from","get_namespace","init","init","init","into [...]
["iceberg_catalog_memory",{"t":"FNNNNNNNNNNNNNNNNNNNNNNNNNNN","n":["MemoryCatalog","borrow","borrow_mut","create_namespace","create_table","deref","deref_mut","drop","drop_namespace","drop_table","fmt","from","get_namespace","init","into","list_namespaces","list_tables","load_table","namespace_exists","new","rename_table","table_exists","try_from","try_into","type_id","update_namespace","update_table","vzip"],"q":[[0,"iceberg_catalog_memory"],[28,"iceberg_catalog_memory::catalog"],[29,"i [...]
diff --git a/api/search.desc/iceberg/iceberg-desc-0-.js b/api/search.desc/iceberg/iceberg-desc-0-.js
index 406c58e7..a943702c 100644
--- a/api/search.desc/iceberg/iceberg-desc-0-.js
+++ b/api/search.desc/iceberg/iceberg-desc-0-.js
@@ -1 +1 @@
-searchState.loadedDescShard("iceberg", 0, "Apache Iceberg Official Native Rust Implementation\nAdd a new schema to the table\nAdd snapshot to table.\nAdd sort order to table.\nAdd a new partition spec to the table\nAssign a new UUID to the table\nThe catalog API for Iceberg Rust.\nThe table’s current schema id must match the requirement.\nIceberg data is invalid.\nThe table’s default sort order id must match the …\nThe table’s default spec id must match the requirement.\nContains the err [...]
\ No newline at end of file
+searchState.loadedDescShard("iceberg", 0, "Apache Iceberg Official Native Rust Implementation\nAdd a new schema to the table\nAdd snapshot to table.\nAdd sort order to table.\nAdd a new partition spec to the table\nAssign a new UUID to the table\nThe catalog API for Iceberg Rust.\nThe table’s current schema id must match the requirement.\nIceberg data is invalid.\nThe table’s default sort order id must match the …\nThe table’s default spec id must match the requirement.\nContains the err [...]
\ No newline at end of file
diff --git a/api/src-files.js b/api/src-files.js
index 4679ebbf..52f64881 100644
--- a/api/src-files.js
+++ b/api/src-files.js
@@ -1,5 +1,5 @@
var srcIndex = new Map(JSON.parse('[\
-["iceberg",["",[["arrow",[],["mod.rs","reader.rs","schema.rs"]],["avro",[],["mod.rs","schema.rs"]],["catalog",[],["mod.rs"]],["expr",[["visitors",[],["bound_predicate_visitor.rs","expression_evaluator.rs","inclusive_metrics_evaluator.rs","inclusive_projection.rs","manifest_evaluator.rs","mod.rs","page_index_evaluator.rs","row_group_metrics_evaluator.rs"]]],["accessor.rs","mod.rs","predicate.rs","term.rs"]],["io",[],["file_io.rs","mod.rs","object_cache.rs","storage.rs","storage_fs.rs","st [...]
+["iceberg",["",[["arrow",[],["mod.rs","reader.rs","record_batch_transformer.rs","schema.rs"]],["avro",[],["mod.rs","schema.rs"]],["catalog",[],["mod.rs"]],["expr",[["visitors",[],["bound_predicate_visitor.rs","expression_evaluator.rs","inclusive_metrics_evaluator.rs","inclusive_projection.rs","manifest_evaluator.rs","mod.rs","page_index_evaluator.rs","row_group_metrics_evaluator.rs"]]],["accessor.rs","mod.rs","predicate.rs","term.rs"]],["io",[],["file_io.rs","mod.rs","object_cache.rs","s [...]
["iceberg_catalog_glue",["",[],["catalog.rs","error.rs","lib.rs","schema.rs","utils.rs"]]],\
["iceberg_catalog_hms",["",[],["catalog.rs","error.rs","lib.rs","schema.rs","utils.rs"]]],\
["iceberg_catalog_memory",["",[],["catalog.rs","lib.rs","namespace_state.rs"]]],\
diff --git a/api/src/iceberg/arrow/mod.rs.html b/api/src/iceberg/arrow/mod.rs.html
index 5ced15dc..8d0ecb03 100644
--- a/api/src/iceberg/arrow/mod.rs.html
+++ b/api/src/iceberg/arrow/mod.rs.html
@@ -21,6 +21,8 @@
<a href="#21" id="21">21</a>
<a href="#22" id="22">22</a>
<a href="#23" id="23">23</a>
+<a href="#24" id="24">24</a>
+<a href="#25" id="25">25</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -43,5 +45,7 @@
</span><span class="kw">mod </span>schema;
<span class="kw">pub use </span>schema::<span class="kw-2">*</span>;
<span class="kw">mod </span>reader;
+<span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>record_batch_transformer;
+
<span class="kw">pub use </span>reader::<span class="kw-2">*</span>;
</code></pre></div></section></main></body></html>
\ No newline at end of file
diff --git a/api/src/iceberg/arrow/reader.rs.html b/api/src/iceberg/arrow/reader.rs.html
index 327a291f..33769b57 100644
--- a/api/src/iceberg/arrow/reader.rs.html
+++ b/api/src/iceberg/arrow/reader.rs.html
@@ -1172,6 +1172,15 @@
<a href="#1172" id="1172">1172</a>
<a href="#1173" id="1173">1173</a>
<a href="#1174" id="1174">1174</a>
+<a href="#1175" id="1175">1175</a>
+<a href="#1176" id="1176">1176</a>
+<a href="#1177" id="1177">1177</a>
+<a href="#1178" id="1178">1178</a>
+<a href="#1179" id="1179">1179</a>
+<a href="#1180" id="1180">1180</a>
+<a href="#1181" id="1181">1181</a>
+<a href="#1182" id="1182">1182</a>
+<a href="#1183" id="1183">1183</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -1212,6 +1221,7 @@
<span class="kw">use </span>parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
<span class="kw">use </span>parquet::schema::types::{SchemaDescriptor, Type <span class="kw">as </span>ParquetType};
+<span class="kw">use </span><span class="kw">crate</span>::arrow::record_batch_transformer::RecordBatchTransformer;
<span class="kw">use </span><span class="kw">crate</span>::arrow::{arrow_schema_to_schema, get_arrow_datum};
<span class="kw">use </span><span class="kw">crate</span>::error::Result;
<span class="kw">use </span><span class="kw">crate</span>::expr::visitors::bound_predicate_visitor::{visit, BoundPredicateVisitor};
@@ -1383,6 +1393,12 @@
)<span class="question-mark">?</span>;
record_batch_stream_builder = record_batch_stream_builder.with_projection(projection_mask);
+ <span class="comment">// RecordBatchTransformer performs any required transformations on the RecordBatches
+ // that come back from the file, such as type promotion, default column insertion
+ // and column re-ordering
+ </span><span class="kw">let </span><span class="kw-2">mut </span>record_batch_transformer =
+ RecordBatchTransformer::build(task.schema_ref(), task.project_field_ids());
+
<span class="kw">if let </span><span class="prelude-val">Some</span>(batch_size) = batch_size {
record_batch_stream_builder = record_batch_stream_builder.with_batch_size(batch_size);
}
@@ -1435,8 +1451,10 @@
<span class="comment">// Build the batch stream and send all the RecordBatches that it generates
// to the requester.
</span><span class="kw">let </span><span class="kw-2">mut </span>record_batch_stream = record_batch_stream_builder.build()<span class="question-mark">?</span>;
+
<span class="kw">while let </span><span class="prelude-val">Some</span>(batch) = record_batch_stream.try_next().<span class="kw">await</span><span class="question-mark">? </span>{
- tx.send(<span class="prelude-val">Ok</span>(batch)).<span class="kw">await</span><span class="question-mark">?
+ tx.send(record_batch_transformer.process_record_batch(batch))
+ .<span class="kw">await</span><span class="question-mark">?
</span>}
<span class="prelude-val">Ok</span>(())
diff --git a/api/src/iceberg/arrow/record_batch_transformer.rs.html b/api/src/iceberg/arrow/record_batch_transformer.rs.html
new file mode 100644
index 00000000..a8f4f3ef
--- /dev/null
+++ b/api/src/iceberg/arrow/record_batch_transformer.rs.html
@@ -0,0 +1,1245 @@
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `crates/iceberg/src/arrow/record_batch_transformer.rs`."><title>record_batch_transformer.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular [...]
+<a href="#2" id="2">2</a>
+<a href="#3" id="3">3</a>
+<a href="#4" id="4">4</a>
+<a href="#5" id="5">5</a>
+<a href="#6" id="6">6</a>
+<a href="#7" id="7">7</a>
+<a href="#8" id="8">8</a>
+<a href="#9" id="9">9</a>
+<a href="#10" id="10">10</a>
+<a href="#11" id="11">11</a>
+<a href="#12" id="12">12</a>
+<a href="#13" id="13">13</a>
+<a href="#14" id="14">14</a>
+<a href="#15" id="15">15</a>
+<a href="#16" id="16">16</a>
+<a href="#17" id="17">17</a>
+<a href="#18" id="18">18</a>
+<a href="#19" id="19">19</a>
+<a href="#20" id="20">20</a>
+<a href="#21" id="21">21</a>
+<a href="#22" id="22">22</a>
+<a href="#23" id="23">23</a>
+<a href="#24" id="24">24</a>
+<a href="#25" id="25">25</a>
+<a href="#26" id="26">26</a>
+<a href="#27" id="27">27</a>
+<a href="#28" id="28">28</a>
+<a href="#29" id="29">29</a>
+<a href="#30" id="30">30</a>
+<a href="#31" id="31">31</a>
+<a href="#32" id="32">32</a>
+<a href="#33" id="33">33</a>
+<a href="#34" id="34">34</a>
+<a href="#35" id="35">35</a>
+<a href="#36" id="36">36</a>
+<a href="#37" id="37">37</a>
+<a href="#38" id="38">38</a>
+<a href="#39" id="39">39</a>
+<a href="#40" id="40">40</a>
+<a href="#41" id="41">41</a>
+<a href="#42" id="42">42</a>
+<a href="#43" id="43">43</a>
+<a href="#44" id="44">44</a>
+<a href="#45" id="45">45</a>
+<a href="#46" id="46">46</a>
+<a href="#47" id="47">47</a>
+<a href="#48" id="48">48</a>
+<a href="#49" id="49">49</a>
+<a href="#50" id="50">50</a>
+<a href="#51" id="51">51</a>
+<a href="#52" id="52">52</a>
+<a href="#53" id="53">53</a>
+<a href="#54" id="54">54</a>
+<a href="#55" id="55">55</a>
+<a href="#56" id="56">56</a>
+<a href="#57" id="57">57</a>
+<a href="#58" id="58">58</a>
+<a href="#59" id="59">59</a>
+<a href="#60" id="60">60</a>
+<a href="#61" id="61">61</a>
+<a href="#62" id="62">62</a>
+<a href="#63" id="63">63</a>
+<a href="#64" id="64">64</a>
+<a href="#65" id="65">65</a>
+<a href="#66" id="66">66</a>
+<a href="#67" id="67">67</a>
+<a href="#68" id="68">68</a>
+<a href="#69" id="69">69</a>
+<a href="#70" id="70">70</a>
+<a href="#71" id="71">71</a>
+<a href="#72" id="72">72</a>
+<a href="#73" id="73">73</a>
+<a href="#74" id="74">74</a>
+<a href="#75" id="75">75</a>
+<a href="#76" id="76">76</a>
+<a href="#77" id="77">77</a>
+<a href="#78" id="78">78</a>
+<a href="#79" id="79">79</a>
+<a href="#80" id="80">80</a>
+<a href="#81" id="81">81</a>
+<a href="#82" id="82">82</a>
+<a href="#83" id="83">83</a>
+<a href="#84" id="84">84</a>
+<a href="#85" id="85">85</a>
+<a href="#86" id="86">86</a>
+<a href="#87" id="87">87</a>
+<a href="#88" id="88">88</a>
+<a href="#89" id="89">89</a>
+<a href="#90" id="90">90</a>
+<a href="#91" id="91">91</a>
+<a href="#92" id="92">92</a>
+<a href="#93" id="93">93</a>
+<a href="#94" id="94">94</a>
+<a href="#95" id="95">95</a>
+<a href="#96" id="96">96</a>
+<a href="#97" id="97">97</a>
+<a href="#98" id="98">98</a>
+<a href="#99" id="99">99</a>
+<a href="#100" id="100">100</a>
+<a href="#101" id="101">101</a>
+<a href="#102" id="102">102</a>
+<a href="#103" id="103">103</a>
+<a href="#104" id="104">104</a>
+<a href="#105" id="105">105</a>
+<a href="#106" id="106">106</a>
+<a href="#107" id="107">107</a>
+<a href="#108" id="108">108</a>
+<a href="#109" id="109">109</a>
+<a href="#110" id="110">110</a>
+<a href="#111" id="111">111</a>
+<a href="#112" id="112">112</a>
+<a href="#113" id="113">113</a>
+<a href="#114" id="114">114</a>
+<a href="#115" id="115">115</a>
+<a href="#116" id="116">116</a>
+<a href="#117" id="117">117</a>
+<a href="#118" id="118">118</a>
+<a href="#119" id="119">119</a>
+<a href="#120" id="120">120</a>
+<a href="#121" id="121">121</a>
+<a href="#122" id="122">122</a>
+<a href="#123" id="123">123</a>
+<a href="#124" id="124">124</a>
+<a href="#125" id="125">125</a>
+<a href="#126" id="126">126</a>
+<a href="#127" id="127">127</a>
+<a href="#128" id="128">128</a>
+<a href="#129" id="129">129</a>
+<a href="#130" id="130">130</a>
+<a href="#131" id="131">131</a>
+<a href="#132" id="132">132</a>
+<a href="#133" id="133">133</a>
+<a href="#134" id="134">134</a>
+<a href="#135" id="135">135</a>
+<a href="#136" id="136">136</a>
+<a href="#137" id="137">137</a>
+<a href="#138" id="138">138</a>
+<a href="#139" id="139">139</a>
+<a href="#140" id="140">140</a>
+<a href="#141" id="141">141</a>
+<a href="#142" id="142">142</a>
+<a href="#143" id="143">143</a>
+<a href="#144" id="144">144</a>
+<a href="#145" id="145">145</a>
+<a href="#146" id="146">146</a>
+<a href="#147" id="147">147</a>
+<a href="#148" id="148">148</a>
+<a href="#149" id="149">149</a>
+<a href="#150" id="150">150</a>
+<a href="#151" id="151">151</a>
+<a href="#152" id="152">152</a>
+<a href="#153" id="153">153</a>
+<a href="#154" id="154">154</a>
+<a href="#155" id="155">155</a>
+<a href="#156" id="156">156</a>
+<a href="#157" id="157">157</a>
+<a href="#158" id="158">158</a>
+<a href="#159" id="159">159</a>
+<a href="#160" id="160">160</a>
+<a href="#161" id="161">161</a>
+<a href="#162" id="162">162</a>
+<a href="#163" id="163">163</a>
+<a href="#164" id="164">164</a>
+<a href="#165" id="165">165</a>
+<a href="#166" id="166">166</a>
+<a href="#167" id="167">167</a>
+<a href="#168" id="168">168</a>
+<a href="#169" id="169">169</a>
+<a href="#170" id="170">170</a>
+<a href="#171" id="171">171</a>
+<a href="#172" id="172">172</a>
+<a href="#173" id="173">173</a>
+<a href="#174" id="174">174</a>
+<a href="#175" id="175">175</a>
+<a href="#176" id="176">176</a>
+<a href="#177" id="177">177</a>
+<a href="#178" id="178">178</a>
+<a href="#179" id="179">179</a>
+<a href="#180" id="180">180</a>
+<a href="#181" id="181">181</a>
+<a href="#182" id="182">182</a>
+<a href="#183" id="183">183</a>
+<a href="#184" id="184">184</a>
+<a href="#185" id="185">185</a>
+<a href="#186" id="186">186</a>
+<a href="#187" id="187">187</a>
+<a href="#188" id="188">188</a>
+<a href="#189" id="189">189</a>
+<a href="#190" id="190">190</a>
+<a href="#191" id="191">191</a>
+<a href="#192" id="192">192</a>
+<a href="#193" id="193">193</a>
+<a href="#194" id="194">194</a>
+<a href="#195" id="195">195</a>
+<a href="#196" id="196">196</a>
+<a href="#197" id="197">197</a>
+<a href="#198" id="198">198</a>
+<a href="#199" id="199">199</a>
+<a href="#200" id="200">200</a>
+<a href="#201" id="201">201</a>
+<a href="#202" id="202">202</a>
+<a href="#203" id="203">203</a>
+<a href="#204" id="204">204</a>
+<a href="#205" id="205">205</a>
+<a href="#206" id="206">206</a>
+<a href="#207" id="207">207</a>
+<a href="#208" id="208">208</a>
+<a href="#209" id="209">209</a>
+<a href="#210" id="210">210</a>
+<a href="#211" id="211">211</a>
+<a href="#212" id="212">212</a>
+<a href="#213" id="213">213</a>
+<a href="#214" id="214">214</a>
+<a href="#215" id="215">215</a>
+<a href="#216" id="216">216</a>
+<a href="#217" id="217">217</a>
+<a href="#218" id="218">218</a>
+<a href="#219" id="219">219</a>
+<a href="#220" id="220">220</a>
+<a href="#221" id="221">221</a>
+<a href="#222" id="222">222</a>
+<a href="#223" id="223">223</a>
+<a href="#224" id="224">224</a>
+<a href="#225" id="225">225</a>
+<a href="#226" id="226">226</a>
+<a href="#227" id="227">227</a>
+<a href="#228" id="228">228</a>
+<a href="#229" id="229">229</a>
+<a href="#230" id="230">230</a>
+<a href="#231" id="231">231</a>
+<a href="#232" id="232">232</a>
+<a href="#233" id="233">233</a>
+<a href="#234" id="234">234</a>
+<a href="#235" id="235">235</a>
+<a href="#236" id="236">236</a>
+<a href="#237" id="237">237</a>
+<a href="#238" id="238">238</a>
+<a href="#239" id="239">239</a>
+<a href="#240" id="240">240</a>
+<a href="#241" id="241">241</a>
+<a href="#242" id="242">242</a>
+<a href="#243" id="243">243</a>
+<a href="#244" id="244">244</a>
+<a href="#245" id="245">245</a>
+<a href="#246" id="246">246</a>
+<a href="#247" id="247">247</a>
+<a href="#248" id="248">248</a>
+<a href="#249" id="249">249</a>
+<a href="#250" id="250">250</a>
+<a href="#251" id="251">251</a>
+<a href="#252" id="252">252</a>
+<a href="#253" id="253">253</a>
+<a href="#254" id="254">254</a>
+<a href="#255" id="255">255</a>
+<a href="#256" id="256">256</a>
+<a href="#257" id="257">257</a>
+<a href="#258" id="258">258</a>
+<a href="#259" id="259">259</a>
+<a href="#260" id="260">260</a>
+<a href="#261" id="261">261</a>
+<a href="#262" id="262">262</a>
+<a href="#263" id="263">263</a>
+<a href="#264" id="264">264</a>
+<a href="#265" id="265">265</a>
+<a href="#266" id="266">266</a>
+<a href="#267" id="267">267</a>
+<a href="#268" id="268">268</a>
+<a href="#269" id="269">269</a>
+<a href="#270" id="270">270</a>
+<a href="#271" id="271">271</a>
+<a href="#272" id="272">272</a>
+<a href="#273" id="273">273</a>
+<a href="#274" id="274">274</a>
+<a href="#275" id="275">275</a>
+<a href="#276" id="276">276</a>
+<a href="#277" id="277">277</a>
+<a href="#278" id="278">278</a>
+<a href="#279" id="279">279</a>
+<a href="#280" id="280">280</a>
+<a href="#281" id="281">281</a>
+<a href="#282" id="282">282</a>
+<a href="#283" id="283">283</a>
+<a href="#284" id="284">284</a>
+<a href="#285" id="285">285</a>
+<a href="#286" id="286">286</a>
+<a href="#287" id="287">287</a>
+<a href="#288" id="288">288</a>
+<a href="#289" id="289">289</a>
+<a href="#290" id="290">290</a>
+<a href="#291" id="291">291</a>
+<a href="#292" id="292">292</a>
+<a href="#293" id="293">293</a>
+<a href="#294" id="294">294</a>
+<a href="#295" id="295">295</a>
+<a href="#296" id="296">296</a>
+<a href="#297" id="297">297</a>
+<a href="#298" id="298">298</a>
+<a href="#299" id="299">299</a>
+<a href="#300" id="300">300</a>
+<a href="#301" id="301">301</a>
+<a href="#302" id="302">302</a>
+<a href="#303" id="303">303</a>
+<a href="#304" id="304">304</a>
+<a href="#305" id="305">305</a>
+<a href="#306" id="306">306</a>
+<a href="#307" id="307">307</a>
+<a href="#308" id="308">308</a>
+<a href="#309" id="309">309</a>
+<a href="#310" id="310">310</a>
+<a href="#311" id="311">311</a>
+<a href="#312" id="312">312</a>
+<a href="#313" id="313">313</a>
+<a href="#314" id="314">314</a>
+<a href="#315" id="315">315</a>
+<a href="#316" id="316">316</a>
+<a href="#317" id="317">317</a>
+<a href="#318" id="318">318</a>
+<a href="#319" id="319">319</a>
+<a href="#320" id="320">320</a>
+<a href="#321" id="321">321</a>
+<a href="#322" id="322">322</a>
+<a href="#323" id="323">323</a>
+<a href="#324" id="324">324</a>
+<a href="#325" id="325">325</a>
+<a href="#326" id="326">326</a>
+<a href="#327" id="327">327</a>
+<a href="#328" id="328">328</a>
+<a href="#329" id="329">329</a>
+<a href="#330" id="330">330</a>
+<a href="#331" id="331">331</a>
+<a href="#332" id="332">332</a>
+<a href="#333" id="333">333</a>
+<a href="#334" id="334">334</a>
+<a href="#335" id="335">335</a>
+<a href="#336" id="336">336</a>
+<a href="#337" id="337">337</a>
+<a href="#338" id="338">338</a>
+<a href="#339" id="339">339</a>
+<a href="#340" id="340">340</a>
+<a href="#341" id="341">341</a>
+<a href="#342" id="342">342</a>
+<a href="#343" id="343">343</a>
+<a href="#344" id="344">344</a>
+<a href="#345" id="345">345</a>
+<a href="#346" id="346">346</a>
+<a href="#347" id="347">347</a>
+<a href="#348" id="348">348</a>
+<a href="#349" id="349">349</a>
+<a href="#350" id="350">350</a>
+<a href="#351" id="351">351</a>
+<a href="#352" id="352">352</a>
+<a href="#353" id="353">353</a>
+<a href="#354" id="354">354</a>
+<a href="#355" id="355">355</a>
+<a href="#356" id="356">356</a>
+<a href="#357" id="357">357</a>
+<a href="#358" id="358">358</a>
+<a href="#359" id="359">359</a>
+<a href="#360" id="360">360</a>
+<a href="#361" id="361">361</a>
+<a href="#362" id="362">362</a>
+<a href="#363" id="363">363</a>
+<a href="#364" id="364">364</a>
+<a href="#365" id="365">365</a>
+<a href="#366" id="366">366</a>
+<a href="#367" id="367">367</a>
+<a href="#368" id="368">368</a>
+<a href="#369" id="369">369</a>
+<a href="#370" id="370">370</a>
+<a href="#371" id="371">371</a>
+<a href="#372" id="372">372</a>
+<a href="#373" id="373">373</a>
+<a href="#374" id="374">374</a>
+<a href="#375" id="375">375</a>
+<a href="#376" id="376">376</a>
+<a href="#377" id="377">377</a>
+<a href="#378" id="378">378</a>
+<a href="#379" id="379">379</a>
+<a href="#380" id="380">380</a>
+<a href="#381" id="381">381</a>
+<a href="#382" id="382">382</a>
+<a href="#383" id="383">383</a>
+<a href="#384" id="384">384</a>
+<a href="#385" id="385">385</a>
+<a href="#386" id="386">386</a>
+<a href="#387" id="387">387</a>
+<a href="#388" id="388">388</a>
+<a href="#389" id="389">389</a>
+<a href="#390" id="390">390</a>
+<a href="#391" id="391">391</a>
+<a href="#392" id="392">392</a>
+<a href="#393" id="393">393</a>
+<a href="#394" id="394">394</a>
+<a href="#395" id="395">395</a>
+<a href="#396" id="396">396</a>
+<a href="#397" id="397">397</a>
+<a href="#398" id="398">398</a>
+<a href="#399" id="399">399</a>
+<a href="#400" id="400">400</a>
+<a href="#401" id="401">401</a>
+<a href="#402" id="402">402</a>
+<a href="#403" id="403">403</a>
+<a href="#404" id="404">404</a>
+<a href="#405" id="405">405</a>
+<a href="#406" id="406">406</a>
+<a href="#407" id="407">407</a>
+<a href="#408" id="408">408</a>
+<a href="#409" id="409">409</a>
+<a href="#410" id="410">410</a>
+<a href="#411" id="411">411</a>
+<a href="#412" id="412">412</a>
+<a href="#413" id="413">413</a>
+<a href="#414" id="414">414</a>
+<a href="#415" id="415">415</a>
+<a href="#416" id="416">416</a>
+<a href="#417" id="417">417</a>
+<a href="#418" id="418">418</a>
+<a href="#419" id="419">419</a>
+<a href="#420" id="420">420</a>
+<a href="#421" id="421">421</a>
+<a href="#422" id="422">422</a>
+<a href="#423" id="423">423</a>
+<a href="#424" id="424">424</a>
+<a href="#425" id="425">425</a>
+<a href="#426" id="426">426</a>
+<a href="#427" id="427">427</a>
+<a href="#428" id="428">428</a>
+<a href="#429" id="429">429</a>
+<a href="#430" id="430">430</a>
+<a href="#431" id="431">431</a>
+<a href="#432" id="432">432</a>
+<a href="#433" id="433">433</a>
+<a href="#434" id="434">434</a>
+<a href="#435" id="435">435</a>
+<a href="#436" id="436">436</a>
+<a href="#437" id="437">437</a>
+<a href="#438" id="438">438</a>
+<a href="#439" id="439">439</a>
+<a href="#440" id="440">440</a>
+<a href="#441" id="441">441</a>
+<a href="#442" id="442">442</a>
+<a href="#443" id="443">443</a>
+<a href="#444" id="444">444</a>
+<a href="#445" id="445">445</a>
+<a href="#446" id="446">446</a>
+<a href="#447" id="447">447</a>
+<a href="#448" id="448">448</a>
+<a href="#449" id="449">449</a>
+<a href="#450" id="450">450</a>
+<a href="#451" id="451">451</a>
+<a href="#452" id="452">452</a>
+<a href="#453" id="453">453</a>
+<a href="#454" id="454">454</a>
+<a href="#455" id="455">455</a>
+<a href="#456" id="456">456</a>
+<a href="#457" id="457">457</a>
+<a href="#458" id="458">458</a>
+<a href="#459" id="459">459</a>
+<a href="#460" id="460">460</a>
+<a href="#461" id="461">461</a>
+<a href="#462" id="462">462</a>
+<a href="#463" id="463">463</a>
+<a href="#464" id="464">464</a>
+<a href="#465" id="465">465</a>
+<a href="#466" id="466">466</a>
+<a href="#467" id="467">467</a>
+<a href="#468" id="468">468</a>
+<a href="#469" id="469">469</a>
+<a href="#470" id="470">470</a>
+<a href="#471" id="471">471</a>
+<a href="#472" id="472">472</a>
+<a href="#473" id="473">473</a>
+<a href="#474" id="474">474</a>
+<a href="#475" id="475">475</a>
+<a href="#476" id="476">476</a>
+<a href="#477" id="477">477</a>
+<a href="#478" id="478">478</a>
+<a href="#479" id="479">479</a>
+<a href="#480" id="480">480</a>
+<a href="#481" id="481">481</a>
+<a href="#482" id="482">482</a>
+<a href="#483" id="483">483</a>
+<a href="#484" id="484">484</a>
+<a href="#485" id="485">485</a>
+<a href="#486" id="486">486</a>
+<a href="#487" id="487">487</a>
+<a href="#488" id="488">488</a>
+<a href="#489" id="489">489</a>
+<a href="#490" id="490">490</a>
+<a href="#491" id="491">491</a>
+<a href="#492" id="492">492</a>
+<a href="#493" id="493">493</a>
+<a href="#494" id="494">494</a>
+<a href="#495" id="495">495</a>
+<a href="#496" id="496">496</a>
+<a href="#497" id="497">497</a>
+<a href="#498" id="498">498</a>
+<a href="#499" id="499">499</a>
+<a href="#500" id="500">500</a>
+<a href="#501" id="501">501</a>
+<a href="#502" id="502">502</a>
+<a href="#503" id="503">503</a>
+<a href="#504" id="504">504</a>
+<a href="#505" id="505">505</a>
+<a href="#506" id="506">506</a>
+<a href="#507" id="507">507</a>
+<a href="#508" id="508">508</a>
+<a href="#509" id="509">509</a>
+<a href="#510" id="510">510</a>
+<a href="#511" id="511">511</a>
+<a href="#512" id="512">512</a>
+<a href="#513" id="513">513</a>
+<a href="#514" id="514">514</a>
+<a href="#515" id="515">515</a>
+<a href="#516" id="516">516</a>
+<a href="#517" id="517">517</a>
+<a href="#518" id="518">518</a>
+<a href="#519" id="519">519</a>
+<a href="#520" id="520">520</a>
+<a href="#521" id="521">521</a>
+<a href="#522" id="522">522</a>
+<a href="#523" id="523">523</a>
+<a href="#524" id="524">524</a>
+<a href="#525" id="525">525</a>
+<a href="#526" id="526">526</a>
+<a href="#527" id="527">527</a>
+<a href="#528" id="528">528</a>
+<a href="#529" id="529">529</a>
+<a href="#530" id="530">530</a>
+<a href="#531" id="531">531</a>
+<a href="#532" id="532">532</a>
+<a href="#533" id="533">533</a>
+<a href="#534" id="534">534</a>
+<a href="#535" id="535">535</a>
+<a href="#536" id="536">536</a>
+<a href="#537" id="537">537</a>
+<a href="#538" id="538">538</a>
+<a href="#539" id="539">539</a>
+<a href="#540" id="540">540</a>
+<a href="#541" id="541">541</a>
+<a href="#542" id="542">542</a>
+<a href="#543" id="543">543</a>
+<a href="#544" id="544">544</a>
+<a href="#545" id="545">545</a>
+<a href="#546" id="546">546</a>
+<a href="#547" id="547">547</a>
+<a href="#548" id="548">548</a>
+<a href="#549" id="549">549</a>
+<a href="#550" id="550">550</a>
+<a href="#551" id="551">551</a>
+<a href="#552" id="552">552</a>
+<a href="#553" id="553">553</a>
+<a href="#554" id="554">554</a>
+<a href="#555" id="555">555</a>
+<a href="#556" id="556">556</a>
+<a href="#557" id="557">557</a>
+<a href="#558" id="558">558</a>
+<a href="#559" id="559">559</a>
+<a href="#560" id="560">560</a>
+<a href="#561" id="561">561</a>
+<a href="#562" id="562">562</a>
+<a href="#563" id="563">563</a>
+<a href="#564" id="564">564</a>
+<a href="#565" id="565">565</a>
+<a href="#566" id="566">566</a>
+<a href="#567" id="567">567</a>
+<a href="#568" id="568">568</a>
+<a href="#569" id="569">569</a>
+<a href="#570" id="570">570</a>
+<a href="#571" id="571">571</a>
+<a href="#572" id="572">572</a>
+<a href="#573" id="573">573</a>
+<a href="#574" id="574">574</a>
+<a href="#575" id="575">575</a>
+<a href="#576" id="576">576</a>
+<a href="#577" id="577">577</a>
+<a href="#578" id="578">578</a>
+<a href="#579" id="579">579</a>
+<a href="#580" id="580">580</a>
+<a href="#581" id="581">581</a>
+<a href="#582" id="582">582</a>
+<a href="#583" id="583">583</a>
+<a href="#584" id="584">584</a>
+<a href="#585" id="585">585</a>
+<a href="#586" id="586">586</a>
+<a href="#587" id="587">587</a>
+<a href="#588" id="588">588</a>
+<a href="#589" id="589">589</a>
+<a href="#590" id="590">590</a>
+<a href="#591" id="591">591</a>
+<a href="#592" id="592">592</a>
+<a href="#593" id="593">593</a>
+<a href="#594" id="594">594</a>
+<a href="#595" id="595">595</a>
+<a href="#596" id="596">596</a>
+<a href="#597" id="597">597</a>
+<a href="#598" id="598">598</a>
+<a href="#599" id="599">599</a>
+<a href="#600" id="600">600</a>
+<a href="#601" id="601">601</a>
+<a href="#602" id="602">602</a>
+<a href="#603" id="603">603</a>
+<a href="#604" id="604">604</a>
+<a href="#605" id="605">605</a>
+<a href="#606" id="606">606</a>
+<a href="#607" id="607">607</a>
+<a href="#608" id="608">608</a>
+<a href="#609" id="609">609</a>
+<a href="#610" id="610">610</a>
+<a href="#611" id="611">611</a>
+<a href="#612" id="612">612</a>
+<a href="#613" id="613">613</a>
+<a href="#614" id="614">614</a>
+<a href="#615" id="615">615</a>
+<a href="#616" id="616">616</a>
+<a href="#617" id="617">617</a>
+<a href="#618" id="618">618</a>
+<a href="#619" id="619">619</a>
+<a href="#620" id="620">620</a>
+<a href="#621" id="621">621</a>
+<a href="#622" id="622">622</a>
+</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+</span><span class="kw">use </span>std::collections::HashMap;
+<span class="kw">use </span>std::sync::Arc;
+
+<span class="kw">use </span>arrow_array::{
+ Array <span class="kw">as </span>ArrowArray, ArrayRef, BinaryArray, BooleanArray, Float32Array, Float64Array,
+ Int32Array, Int64Array, NullArray, RecordBatch, StringArray,
+};
+<span class="kw">use </span>arrow_cast::cast;
+<span class="kw">use </span>arrow_schema::{
+ DataType, FieldRef, Schema <span class="kw">as </span>ArrowSchema, SchemaRef <span class="kw">as </span>ArrowSchemaRef, SchemaRef,
+};
+<span class="kw">use </span>parquet::arrow::PARQUET_FIELD_ID_META_KEY;
+
+<span class="kw">use </span><span class="kw">crate</span>::arrow::schema_to_arrow_schema;
+<span class="kw">use </span><span class="kw">crate</span>::spec::{Literal, PrimitiveLiteral, Schema <span class="kw">as </span>IcebergSchema};
+<span class="kw">use crate</span>::{Error, ErrorKind, <span class="prelude-ty">Result</span>};
+
+<span class="doccomment">/// Indicates how a particular column in a processed RecordBatch should
+/// be sourced.
+</span><span class="attr">#[derive(Debug)]
+</span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">enum </span>ColumnSource {
+ <span class="comment">// signifies that a column should be passed through unmodified
+ // from the file's RecordBatch
+ </span>PassThrough {
+ source_index: usize,
+ },
+
+ <span class="comment">// signifies that a column from the file's RecordBatch has undergone
+ // type promotion so the source column with the given index needs
+ // to be promoted to the specified type
+ </span>Promote {
+ target_type: DataType,
+ source_index: usize,
+ },
+
+ <span class="comment">// Signifies that a new column has been inserted before the column
+ // with index `index`. (we choose "before" rather than "after" so
+ // that we can use usize; if we insert after, then we need to
+ // be able to store -1 here to signify that a new
+ // column is to be added at the front of the column list).
+ // If multiple columns need to be inserted at a given
+ // location, they should all be given the same index, as the index
+ // here refers to the original RecordBatch, not the interim state after
+ // a preceding operation.
+ </span>Add {
+ target_type: DataType,
+ value: <span class="prelude-ty">Option</span><PrimitiveLiteral>,
+ },
+ <span class="comment">// The iceberg spec refers to other permissible schema evolution actions
+ // (see https://iceberg.apache.org/spec/#schema-evolution):
+ // renaming fields, deleting fields and reordering fields.
+ // Renames only affect the schema of the RecordBatch rather than the
+ // columns themselves, so a single updated cached schema can
+ // be re-used and no per-column actions are required.
+ // Deletion and Reorder can be achieved without needing this
+ // post-processing step by using the projection mask.
+</span>}
+
+<span class="attr">#[derive(Debug)]
+</span><span class="kw">enum </span>BatchTransform {
+ <span class="comment">// Indicates that no changes need to be performed to the RecordBatches
+ // coming in from the stream and that they can be passed through
+ // unmodified
+ </span>PassThrough,
+
+ Modify {
+ <span class="comment">// Every transformed RecordBatch will have the same schema. We create the
+ // target just once and cache it here. Helpfully, Arc<Schema> is needed in
+ // the constructor for RecordBatch, so we don't need an expensive copy
+ // each time we build a new RecordBatch
+ </span>target_schema: Arc<ArrowSchema>,
+
+ <span class="comment">// Indicates how each column in the target schema is derived.
+ </span>operations: Vec<ColumnSource>,
+ },
+
+ <span class="comment">// Sometimes only the schema will need modifying, for example when
+ // the column names have changed vs the file, but not the column types.
+ // we can avoid a heap allocation per RecordBach in this case by retaining
+ // the existing column Vec.
+ </span>ModifySchema {
+ target_schema: Arc<ArrowSchema>,
+ },
+}
+
+<span class="attr">#[derive(Debug)]
+</span><span class="kw">enum </span>SchemaComparison {
+ Equivalent,
+ NameChangesOnly,
+ Different,
+}
+
+<span class="attr">#[derive(Debug)]
+</span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">struct </span>RecordBatchTransformer {
+ snapshot_schema: Arc<IcebergSchema>,
+ projected_iceberg_field_ids: Vec<i32>,
+
+ <span class="comment">// BatchTransform gets lazily constructed based on the schema of
+ // the first RecordBatch we receive from the file
+ </span>batch_transform: <span class="prelude-ty">Option</span><BatchTransform>,
+}
+
+<span class="kw">impl </span>RecordBatchTransformer {
+ <span class="doccomment">/// Build a RecordBatchTransformer for a given
+ /// Iceberg snapshot schema and list of projected field ids.
+ </span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">fn </span>build(
+ snapshot_schema: Arc<IcebergSchema>,
+ projected_iceberg_field_ids: <span class="kw-2">&</span>[i32],
+ ) -> <span class="self">Self </span>{
+ <span class="kw">let </span>projected_iceberg_field_ids = <span class="kw">if </span>projected_iceberg_field_ids.is_empty() {
+ <span class="comment">// If the list of field ids is empty, this indicates that we
+ // need to select all fields.
+ // Project all fields in table schema order
+ </span>snapshot_schema
+ .as_struct()
+ .fields()
+ .iter()
+ .map(|field| field.id)
+ .collect()
+ } <span class="kw">else </span>{
+ projected_iceberg_field_ids.to_vec()
+ };
+
+ <span class="self">Self </span>{
+ snapshot_schema,
+ projected_iceberg_field_ids,
+ batch_transform: <span class="prelude-val">None</span>,
+ }
+ }
+
+ <span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">fn </span>process_record_batch(
+ <span class="kw-2">&mut </span><span class="self">self</span>,
+ record_batch: RecordBatch,
+ ) -> <span class="prelude-ty">Result</span><RecordBatch> {
+ <span class="prelude-val">Ok</span>(<span class="kw">match </span><span class="kw-2">&</span><span class="self">self</span>.batch_transform {
+ <span class="prelude-val">Some</span>(BatchTransform::PassThrough) => record_batch,
+ <span class="prelude-val">Some</span>(BatchTransform::Modify {
+ <span class="kw-2">ref </span>target_schema,
+ <span class="kw-2">ref </span>operations,
+ }) => RecordBatch::try_new(
+ target_schema.clone(),
+ <span class="self">self</span>.transform_columns(record_batch.columns(), operations)<span class="question-mark">?</span>,
+ )<span class="question-mark">?</span>,
+ <span class="prelude-val">Some</span>(BatchTransform::ModifySchema { target_schema }) => {
+ record_batch.with_schema(target_schema.clone())<span class="question-mark">?
+ </span>}
+ <span class="prelude-val">None </span>=> {
+ <span class="self">self</span>.batch_transform = <span class="prelude-val">Some</span>(<span class="self">Self</span>::generate_batch_transform(
+ record_batch.schema_ref(),
+ <span class="self">self</span>.snapshot_schema.as_ref(),
+ <span class="kw-2">&</span><span class="self">self</span>.projected_iceberg_field_ids,
+ )<span class="question-mark">?</span>);
+
+ <span class="self">self</span>.process_record_batch(record_batch)<span class="question-mark">?
+ </span>}
+ })
+ }
+
+ <span class="comment">// Compare the schema of the incoming RecordBatches to the schema of
+ // the Iceberg snapshot to determine what, if any, transformation
+ // needs to be applied. If the schemas match, we return BatchTransform::PassThrough
+ // to indicate that no changes need to be made. Otherwise, we return a
+ // BatchTransform::Modify containing the target RecordBatch schema and
+ // the list of `ColumnSource`s that indicate how to source each column in
+ // the resulting RecordBatches.
+ </span><span class="kw">fn </span>generate_batch_transform(
+ source_schema: <span class="kw-2">&</span>ArrowSchemaRef,
+ snapshot_schema: <span class="kw-2">&</span>IcebergSchema,
+ projected_iceberg_field_ids: <span class="kw-2">&</span>[i32],
+ ) -> <span class="prelude-ty">Result</span><BatchTransform> {
+ <span class="kw">let </span>mapped_unprojected_arrow_schema = Arc::new(schema_to_arrow_schema(snapshot_schema)<span class="question-mark">?</span>);
+ <span class="kw">let </span>field_id_to_mapped_schema_map =
+ <span class="self">Self</span>::build_field_id_to_arrow_schema_map(<span class="kw-2">&</span>mapped_unprojected_arrow_schema)<span class="question-mark">?</span>;
+
+ <span class="comment">// Create a new arrow schema by selecting fields from mapped_unprojected,
+ // in the order of the field ids in projected_iceberg_field_ids
+ </span><span class="kw">let </span>fields: <span class="prelude-ty">Result</span><Vec<<span class="kw">_</span>>> = projected_iceberg_field_ids
+ .iter()
+ .map(|field_id| {
+ <span class="prelude-val">Ok</span>(field_id_to_mapped_schema_map
+ .get(field_id)
+ .ok_or(Error::new(ErrorKind::Unexpected, <span class="string">"field not found"</span>))<span class="question-mark">?
+ </span>.<span class="number">0
+ </span>.clone())
+ })
+ .collect();
+
+ <span class="kw">let </span>target_schema = Arc::new(ArrowSchema::new(fields<span class="question-mark">?</span>));
+
+ <span class="kw">match </span><span class="self">Self</span>::compare_schemas(source_schema, <span class="kw-2">&</span>target_schema) {
+ SchemaComparison::Equivalent => <span class="prelude-val">Ok</span>(BatchTransform::PassThrough),
+ SchemaComparison::NameChangesOnly => <span class="prelude-val">Ok</span>(BatchTransform::ModifySchema { target_schema }),
+ SchemaComparison::Different => <span class="prelude-val">Ok</span>(BatchTransform::Modify {
+ operations: <span class="self">Self</span>::generate_transform_operations(
+ source_schema,
+ snapshot_schema,
+ projected_iceberg_field_ids,
+ field_id_to_mapped_schema_map,
+ )<span class="question-mark">?</span>,
+ target_schema,
+ }),
+ }
+ }
+
+ <span class="doccomment">/// Compares the source and target schemas
+ /// Determines if they have changed in any meaningful way:
+ /// * If they have different numbers of fields, then we need to modify
+ /// the incoming RecordBatch schema AND columns
+ /// * If they have the same number of fields, but some of them differ in
+ /// either data type or nullability, then we need to modify the
+ /// incoming RecordBatch schema AND columns
+ /// * If the schemas differ only in the column names, then we need
+ /// to modify the RecordBatch schema BUT we can keep the
+ /// original column data unmodified
+ /// * If the schemas are identical (or differ only in inconsequential
+ /// ways) then we can pass through the original RecordBatch unmodified
+ </span><span class="kw">fn </span>compare_schemas(
+ source_schema: <span class="kw-2">&</span>ArrowSchemaRef,
+ target_schema: <span class="kw-2">&</span>ArrowSchemaRef,
+ ) -> SchemaComparison {
+ <span class="kw">if </span>source_schema.fields().len() != target_schema.fields().len() {
+ <span class="kw">return </span>SchemaComparison::Different;
+ }
+
+ <span class="kw">let </span><span class="kw-2">mut </span>names_changed = <span class="bool-val">false</span>;
+
+ <span class="kw">for </span>(source_field, target_field) <span class="kw">in </span>source_schema
+ .fields()
+ .iter()
+ .zip(target_schema.fields().iter())
+ {
+ <span class="kw">if </span>source_field.data_type() != target_field.data_type()
+ || source_field.is_nullable() != target_field.is_nullable()
+ {
+ <span class="kw">return </span>SchemaComparison::Different;
+ }
+
+ <span class="kw">if </span>source_field.name() != target_field.name() {
+ names_changed = <span class="bool-val">true</span>;
+ }
+ }
+
+ <span class="kw">if </span>names_changed {
+ SchemaComparison::NameChangesOnly
+ } <span class="kw">else </span>{
+ SchemaComparison::Equivalent
+ }
+ }
+
+ <span class="kw">fn </span>generate_transform_operations(
+ source_schema: <span class="kw-2">&</span>ArrowSchemaRef,
+ snapshot_schema: <span class="kw-2">&</span>IcebergSchema,
+ projected_iceberg_field_ids: <span class="kw-2">&</span>[i32],
+ field_id_to_mapped_schema_map: HashMap<i32, (FieldRef, usize)>,
+ ) -> <span class="prelude-ty">Result</span><Vec<ColumnSource>> {
+ <span class="kw">let </span>field_id_to_source_schema_map =
+ <span class="self">Self</span>::build_field_id_to_arrow_schema_map(source_schema)<span class="question-mark">?</span>;
+
+ projected_iceberg_field_ids.iter().map(|field_id|{
+ <span class="kw">let </span>(target_field, <span class="kw">_</span>) = field_id_to_mapped_schema_map.get(field_id).ok_or(
+ Error::new(ErrorKind::Unexpected, <span class="string">"could not find field in schema"</span>)
+ )<span class="question-mark">?</span>;
+ <span class="kw">let </span>target_type = target_field.data_type();
+
+ <span class="prelude-val">Ok</span>(<span class="kw">if let </span><span class="prelude-val">Some</span>((source_field, source_index)) = field_id_to_source_schema_map.get(field_id) {
+ <span class="comment">// column present in source
+
+ </span><span class="kw">if </span>source_field.data_type().equals_datatype(target_type) {
+ <span class="comment">// no promotion required
+ </span>ColumnSource::PassThrough {
+ source_index: <span class="kw-2">*</span>source_index
+ }
+ } <span class="kw">else </span>{
+ <span class="comment">// promotion required
+ </span>ColumnSource::Promote {
+ target_type: target_type.clone(),
+ source_index: <span class="kw-2">*</span>source_index,
+ }
+ }
+ } <span class="kw">else </span>{
+ <span class="comment">// column must be added
+ </span><span class="kw">let </span>iceberg_field = snapshot_schema.field_by_id(<span class="kw-2">*</span>field_id).ok_or(
+ Error::new(ErrorKind::Unexpected, <span class="string">"Field not found in snapshot schema"</span>)
+ )<span class="question-mark">?</span>;
+
+ <span class="kw">let </span>default_value = <span class="kw">if let </span><span class="prelude-val">Some</span>(<span class="kw-2">ref </span>iceberg_default_value) =
+ <span class="kw-2">&</span>iceberg_field.initial_default
+ {
+ <span class="kw">let </span>Literal::Primitive(primitive_literal) = iceberg_default_value <span class="kw">else </span>{
+ <span class="kw">return </span><span class="prelude-val">Err</span>(Error::new(
+ ErrorKind::Unexpected,
+ <span class="macro">format!</span>(<span class="string">"Default value for column must be primitive type, but encountered {:?}"</span>, iceberg_default_value)
+ ));
+ };
+ <span class="prelude-val">Some</span>(primitive_literal.clone())
+ } <span class="kw">else </span>{
+ <span class="prelude-val">None
+ </span>};
+
+ ColumnSource::Add {
+ value: default_value,
+ target_type: target_type.clone(),
+ }
+ })
+ }).collect()
+ }
+
+ <span class="kw">fn </span>build_field_id_to_arrow_schema_map(
+ source_schema: <span class="kw-2">&</span>SchemaRef,
+ ) -> <span class="prelude-ty">Result</span><HashMap<i32, (FieldRef, usize)>> {
+ <span class="kw">let </span><span class="kw-2">mut </span>field_id_to_source_schema = HashMap::new();
+ <span class="kw">for </span>(source_field_idx, source_field) <span class="kw">in </span>source_schema.fields.iter().enumerate() {
+ <span class="kw">let </span>this_field_id = source_field
+ .metadata()
+ .get(PARQUET_FIELD_ID_META_KEY)
+ .ok_or_else(|| {
+ Error::new(
+ ErrorKind::DataInvalid,
+ <span class="string">"field ID not present in parquet metadata"</span>,
+ )
+ })<span class="question-mark">?
+ </span>.parse()
+ .map_err(|e| {
+ Error::new(
+ ErrorKind::DataInvalid,
+ <span class="macro">format!</span>(<span class="string">"field id not parseable as an i32: {}"</span>, e),
+ )
+ })<span class="question-mark">?</span>;
+
+ field_id_to_source_schema
+ .insert(this_field_id, (source_field.clone(), source_field_idx));
+ }
+
+ <span class="prelude-val">Ok</span>(field_id_to_source_schema)
+ }
+
+ <span class="kw">fn </span>transform_columns(
+ <span class="kw-2">&</span><span class="self">self</span>,
+ columns: <span class="kw-2">&</span>[Arc<<span class="kw">dyn </span>ArrowArray>],
+ operations: <span class="kw-2">&</span>[ColumnSource],
+ ) -> <span class="prelude-ty">Result</span><Vec<Arc<<span class="kw">dyn </span>ArrowArray>>> {
+ <span class="kw">if </span>columns.is_empty() {
+ <span class="kw">return </span><span class="prelude-val">Ok</span>(columns.to_vec());
+ }
+ <span class="kw">let </span>num_rows = columns[<span class="number">0</span>].len();
+
+ operations
+ .iter()
+ .map(|op| {
+ <span class="prelude-val">Ok</span>(<span class="kw">match </span>op {
+ ColumnSource::PassThrough { source_index } => columns[<span class="kw-2">*</span>source_index].clone(),
+
+ ColumnSource::Promote {
+ target_type,
+ source_index,
+ } => cast(<span class="kw-2">&*</span>columns[<span class="kw-2">*</span>source_index], target_type)<span class="question-mark">?</span>,
+
+ ColumnSource::Add { target_type, value } => {
+ <span class="self">Self</span>::create_column(target_type, value, num_rows)<span class="question-mark">?
+ </span>}
+ })
+ })
+ .collect()
+ }
+
+ <span class="kw">fn </span>create_column(
+ target_type: <span class="kw-2">&</span>DataType,
+ prim_lit: <span class="kw-2">&</span><span class="prelude-ty">Option</span><PrimitiveLiteral>,
+ num_rows: usize,
+ ) -> <span class="prelude-ty">Result</span><ArrayRef> {
+ <span class="prelude-val">Ok</span>(<span class="kw">match </span>(target_type, prim_lit) {
+ (DataType::Boolean, <span class="prelude-val">Some</span>(PrimitiveLiteral::Boolean(value))) => {
+ Arc::new(BooleanArray::from(<span class="macro">vec!</span>[<span class="kw-2">*</span>value; num_rows]))
+ }
+ (DataType::Boolean, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><bool>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(BooleanArray::from(vals))
+ }
+ (DataType::Int32, <span class="prelude-val">Some</span>(PrimitiveLiteral::Int(value))) => {
+ Arc::new(Int32Array::from(<span class="macro">vec!</span>[<span class="kw-2">*</span>value; num_rows]))
+ }
+ (DataType::Int32, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><i32>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(Int32Array::from(vals))
+ }
+ (DataType::Int64, <span class="prelude-val">Some</span>(PrimitiveLiteral::Long(value))) => {
+ Arc::new(Int64Array::from(<span class="macro">vec!</span>[<span class="kw-2">*</span>value; num_rows]))
+ }
+ (DataType::Int64, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><i64>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(Int64Array::from(vals))
+ }
+ (DataType::Float32, <span class="prelude-val">Some</span>(PrimitiveLiteral::Float(value))) => {
+ Arc::new(Float32Array::from(<span class="macro">vec!</span>[value.<span class="number">0</span>; num_rows]))
+ }
+ (DataType::Float32, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><f32>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(Float32Array::from(vals))
+ }
+ (DataType::Float64, <span class="prelude-val">Some</span>(PrimitiveLiteral::Double(value))) => {
+ Arc::new(Float64Array::from(<span class="macro">vec!</span>[value.<span class="number">0</span>; num_rows]))
+ }
+ (DataType::Float64, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><f64>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(Float64Array::from(vals))
+ }
+ (DataType::Utf8, <span class="prelude-val">Some</span>(PrimitiveLiteral::String(value))) => {
+ Arc::new(StringArray::from(<span class="macro">vec!</span>[value.clone(); num_rows]))
+ }
+ (DataType::Utf8, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><String>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(StringArray::from(vals))
+ }
+ (DataType::Binary, <span class="prelude-val">Some</span>(PrimitiveLiteral::Binary(value))) => {
+ Arc::new(BinaryArray::from_vec(<span class="macro">vec!</span>[value; num_rows]))
+ }
+ (DataType::Binary, <span class="prelude-val">None</span>) => {
+ <span class="kw">let </span>vals: Vec<<span class="prelude-ty">Option</span><<span class="kw-2">&</span>[u8]>> = <span class="macro">vec!</span>[<span class="prelude-val">None</span>; num_rows];
+ Arc::new(BinaryArray::from_opt_vec(vals))
+ }
+ (DataType::Null, <span class="kw">_</span>) => Arc::new(NullArray::new(num_rows)),
+ (dt, <span class="kw">_</span>) => {
+ <span class="kw">return </span><span class="prelude-val">Err</span>(Error::new(
+ ErrorKind::Unexpected,
+ <span class="macro">format!</span>(<span class="string">"unexpected target column type {}"</span>, dt),
+ ))
+ }
+ })
+ }
+}
+
+<span class="attr">#[cfg(test)]
+</span><span class="kw">mod </span>test {
+ <span class="kw">use </span>std::collections::HashMap;
+ <span class="kw">use </span>std::sync::Arc;
+
+ <span class="kw">use </span>arrow_array::{
+ Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray,
+ };
+ <span class="kw">use </span>arrow_schema::{DataType, Field, Schema <span class="kw">as </span>ArrowSchema};
+ <span class="kw">use </span>parquet::arrow::PARQUET_FIELD_ID_META_KEY;
+
+ <span class="kw">use </span><span class="kw">crate</span>::arrow::record_batch_transformer::RecordBatchTransformer;
+ <span class="kw">use </span><span class="kw">crate</span>::spec::{Literal, NestedField, PrimitiveType, Schema, Type};
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>build_field_id_to_source_schema_map_works() {
+ <span class="kw">let </span>arrow_schema = arrow_schema_already_same_as_target();
+
+ <span class="kw">let </span>result =
+ RecordBatchTransformer::build_field_id_to_arrow_schema_map(<span class="kw-2">&</span>arrow_schema).unwrap();
+
+ <span class="kw">let </span>expected = HashMap::from_iter([
+ (<span class="number">10</span>, (arrow_schema.fields()[<span class="number">0</span>].clone(), <span class="number">0</span>)),
+ (<span class="number">11</span>, (arrow_schema.fields()[<span class="number">1</span>].clone(), <span class="number">1</span>)),
+ (<span class="number">12</span>, (arrow_schema.fields()[<span class="number">2</span>].clone(), <span class="number">2</span>)),
+ (<span class="number">14</span>, (arrow_schema.fields()[<span class="number">3</span>].clone(), <span class="number">3</span>)),
+ (<span class="number">15</span>, (arrow_schema.fields()[<span class="number">4</span>].clone(), <span class="number">4</span>)),
+ ]);
+
+ <span class="macro">assert!</span>(result.eq(<span class="kw-2">&</span>expected));
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>processor_returns_properly_shaped_record_batch_when_no_schema_migration_required() {
+ <span class="kw">let </span>snapshot_schema = Arc::new(iceberg_table_schema());
+ <span class="kw">let </span>projected_iceberg_field_ids = [<span class="number">13</span>, <span class="number">14</span>];
+
+ <span class="kw">let </span><span class="kw-2">mut </span>inst = RecordBatchTransformer::build(snapshot_schema, <span class="kw-2">&</span>projected_iceberg_field_ids);
+
+ <span class="kw">let </span>result = inst
+ .process_record_batch(source_record_batch_no_migration_required())
+ .unwrap();
+
+ <span class="kw">let </span>expected = source_record_batch_no_migration_required();
+
+ <span class="macro">assert_eq!</span>(result, expected);
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>processor_returns_properly_shaped_record_batch_when_schema_migration_required() {
+ <span class="kw">let </span>snapshot_schema = Arc::new(iceberg_table_schema());
+ <span class="kw">let </span>projected_iceberg_field_ids = [<span class="number">10</span>, <span class="number">11</span>, <span class="number">12</span>, <span class="number">14</span>, <span class="number">15</span>]; <span class="comment">// a, b, c, e, f
+
+ </span><span class="kw">let </span><span class="kw-2">mut </span>inst = RecordBatchTransformer::build(snapshot_schema, <span class="kw-2">&</span>projected_iceberg_field_ids);
+
+ <span class="kw">let </span>result = inst.process_record_batch(source_record_batch()).unwrap();
+
+ <span class="kw">let </span>expected = expected_record_batch_migration_required();
+
+ <span class="macro">assert_eq!</span>(result, expected);
+ }
+
+ <span class="kw">pub fn </span>source_record_batch() -> RecordBatch {
+ RecordBatch::try_new(
+ arrow_schema_promotion_addition_and_renaming_required(),
+ <span class="macro">vec!</span>[
+ Arc::new(Int32Array::from(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">1001</span>), <span class="prelude-val">Some</span>(<span class="number">1002</span>), <span class="prelude-val">Some</span>(<span class="number">1003</span>)])), <span class="comment">// b
+ </span>Arc::new(Float32Array::from(<span class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(<span class="number">12.125</span>),
+ <span class="prelude-val">Some</span>(<span class="number">23.375</span>),
+ <span class="prelude-val">Some</span>(<span class="number">34.875</span>),
+ ])), <span class="comment">// c
+ </span>Arc::new(Int32Array::from(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">2001</span>), <span class="prelude-val">Some</span>(<span class="number">2002</span>), <span class="prelude-val">Some</span>(<span class="number">2003</span>)])), <span class="comment">// d
+ </span>Arc::new(StringArray::from(<span class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(<span class="string">"Apache"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"Iceberg"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"Rocks"</span>),
+ ])), <span class="comment">// e
+ </span>],
+ )
+ .unwrap()
+ }
+
+ <span class="kw">pub fn </span>source_record_batch_no_migration_required() -> RecordBatch {
+ RecordBatch::try_new(
+ arrow_schema_no_promotion_addition_or_renaming_required(),
+ <span class="macro">vec!</span>[
+ Arc::new(Int32Array::from(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">2001</span>), <span class="prelude-val">Some</span>(<span class="number">2002</span>), <span class="prelude-val">Some</span>(<span class="number">2003</span>)])), <span class="comment">// d
+ </span>Arc::new(StringArray::from(<span class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(<span class="string">"Apache"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"Iceberg"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"Rocks"</span>),
+ ])), <span class="comment">// e
+ </span>],
+ )
+ .unwrap()
+ }
+
+ <span class="kw">pub fn </span>expected_record_batch_migration_required() -> RecordBatch {
+ RecordBatch::try_new(arrow_schema_already_same_as_target(), <span class="macro">vec!</span>[
+ Arc::new(StringArray::from(Vec::<<span class="prelude-ty">Option</span><String>>::from([
+ <span class="prelude-val">None</span>, <span class="prelude-val">None</span>, <span class="prelude-val">None</span>,
+ ]))), <span class="comment">// a
+ </span>Arc::new(Int64Array::from(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">1001</span>), <span class="prelude-val">Some</span>(<span class="number">1002</span>), <span class="prelude-val">Some</span>(<span class="number">1003</span>)])), <span class="comment">// b
+ </span>Arc::new(Float64Array::from(<span class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(<span class="number">12.125</span>),
+ <span class="prelude-val">Some</span>(<span class="number">23.375</span>),
+ <span class="prelude-val">Some</span>(<span class="number">34.875</span>),
+ ])), <span class="comment">// c
+ </span>Arc::new(StringArray::from(<span class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(<span class="string">"Apache"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"Iceberg"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"Rocks"</span>),
+ ])), <span class="comment">// e (d skipped by projection)
+ </span>Arc::new(StringArray::from(<span class="macro">vec!</span>[
+ <span class="prelude-val">Some</span>(<span class="string">"(╯°□°)╯"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"(╯°□°)╯"</span>),
+ <span class="prelude-val">Some</span>(<span class="string">"(╯°□°)╯"</span>),
+ ])), <span class="comment">// f
+ </span>])
+ .unwrap()
+ }
+
+ <span class="kw">pub fn </span>iceberg_table_schema() -> Schema {
+ Schema::builder()
+ .with_schema_id(<span class="number">2</span>)
+ .with_fields(<span class="macro">vec!</span>[
+ NestedField::optional(<span class="number">10</span>, <span class="string">"a"</span>, Type::Primitive(PrimitiveType::String)).into(),
+ NestedField::required(<span class="number">11</span>, <span class="string">"b"</span>, Type::Primitive(PrimitiveType::Long)).into(),
+ NestedField::required(<span class="number">12</span>, <span class="string">"c"</span>, Type::Primitive(PrimitiveType::Double)).into(),
+ NestedField::required(<span class="number">13</span>, <span class="string">"d"</span>, Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::optional(<span class="number">14</span>, <span class="string">"e"</span>, Type::Primitive(PrimitiveType::String)).into(),
+ NestedField::required(<span class="number">15</span>, <span class="string">"f"</span>, Type::Primitive(PrimitiveType::String))
+ .with_initial_default(Literal::string(<span class="string">"(╯°□°)╯"</span>))
+ .into(),
+ ])
+ .build()
+ .unwrap()
+ }
+
+ <span class="kw">fn </span>arrow_schema_already_same_as_target() -> Arc<ArrowSchema> {
+ Arc::new(ArrowSchema::new(<span class="macro">vec!</span>[
+ simple_field(<span class="string">"a"</span>, DataType::Utf8, <span class="bool-val">true</span>, <span class="string">"10"</span>),
+ simple_field(<span class="string">"b"</span>, DataType::Int64, <span class="bool-val">false</span>, <span class="string">"11"</span>),
+ simple_field(<span class="string">"c"</span>, DataType::Float64, <span class="bool-val">false</span>, <span class="string">"12"</span>),
+ simple_field(<span class="string">"e"</span>, DataType::Utf8, <span class="bool-val">true</span>, <span class="string">"14"</span>),
+ simple_field(<span class="string">"f"</span>, DataType::Utf8, <span class="bool-val">false</span>, <span class="string">"15"</span>),
+ ]))
+ }
+
+ <span class="kw">fn </span>arrow_schema_promotion_addition_and_renaming_required() -> Arc<ArrowSchema> {
+ Arc::new(ArrowSchema::new(<span class="macro">vec!</span>[
+ simple_field(<span class="string">"b"</span>, DataType::Int32, <span class="bool-val">false</span>, <span class="string">"11"</span>),
+ simple_field(<span class="string">"c"</span>, DataType::Float32, <span class="bool-val">false</span>, <span class="string">"12"</span>),
+ simple_field(<span class="string">"d"</span>, DataType::Int32, <span class="bool-val">false</span>, <span class="string">"13"</span>),
+ simple_field(<span class="string">"e_old"</span>, DataType::Utf8, <span class="bool-val">true</span>, <span class="string">"14"</span>),
+ ]))
+ }
+
+ <span class="kw">fn </span>arrow_schema_no_promotion_addition_or_renaming_required() -> Arc<ArrowSchema> {
+ Arc::new(ArrowSchema::new(<span class="macro">vec!</span>[
+ simple_field(<span class="string">"d"</span>, DataType::Int32, <span class="bool-val">false</span>, <span class="string">"13"</span>),
+ simple_field(<span class="string">"e"</span>, DataType::Utf8, <span class="bool-val">true</span>, <span class="string">"14"</span>),
+ ]))
+ }
+
+ <span class="doccomment">/// Create a simple arrow field with metadata.
+ </span><span class="kw">fn </span>simple_field(name: <span class="kw-2">&</span>str, ty: DataType, nullable: bool, value: <span class="kw-2">&</span>str) -> Field {
+ Field::new(name, ty, nullable).with_metadata(HashMap::from([(
+ PARQUET_FIELD_ID_META_KEY.to_string(),
+ value.to_string(),
+ )]))
+ }
+}
+</code></pre></div></section></main></body></html>
\ No newline at end of file
diff --git a/api/src/iceberg/arrow/schema.rs.html b/api/src/iceberg/arrow/schema.rs.html
index e5f21f34..504fe1b4 100644
--- a/api/src/iceberg/arrow/schema.rs.html
+++ b/api/src/iceberg/arrow/schema.rs.html
@@ -1696,7 +1696,7 @@
visitor.schema(schema, results)
}
-<span class="doccomment">/// Convert Arrow schema to ceberg schema.
+<span class="doccomment">/// Convert Arrow schema to Iceberg schema.
</span><span class="kw">pub fn </span>arrow_schema_to_schema(schema: <span class="kw-2">&</span>ArrowSchema) -> <span class="prelude-ty">Result</span><Schema> {
<span class="kw">let </span><span class="kw-2">mut </span>visitor = ArrowSchemaConverter::new();
visit_schema(schema, <span class="kw-2">&mut </span>visitor)
diff --git a/api/src/iceberg/error.rs.html b/api/src/iceberg/error.rs.html
index 74a2e816..745feab2 100644
--- a/api/src/iceberg/error.rs.html
+++ b/api/src/iceberg/error.rs.html
@@ -461,6 +461,12 @@
<a href="#461" id="461">461</a>
<a href="#462" id="462">462</a>
<a href="#463" id="463">463</a>
+<a href="#464" id="464">464</a>
+<a href="#465" id="465">465</a>
+<a href="#466" id="466">466</a>
+<a href="#467" id="467">467</a>
+<a href="#468" id="468">468</a>
+<a href="#469" id="469">469</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -800,6 +806,12 @@
<span class="string">"Failed to send a message to a channel"
</span>);
+<span class="macro">define_from_err!</span>(
+ arrow_schema::ArrowError,
+ ErrorKind::Unexpected,
+ <span class="string">"Arrow Schema Error"
+</span>);
+
<span class="macro">define_from_err!</span>(std::io::Error, ErrorKind::Unexpected, <span class="string">"IO Operation failed"</span>);
<span class="doccomment">/// Converts a timestamp in milliseconds to `DateTime<Utc>`, handling errors.
diff --git a/api/src/iceberg/lib.rs.html b/api/src/iceberg/lib.rs.html
index a0293efc..b776f141 100644
--- a/api/src/iceberg/lib.rs.html
+++ b/api/src/iceberg/lib.rs.html
@@ -83,6 +83,7 @@
<a href="#83" id="83">83</a>
<a href="#84" id="84">84</a>
<a href="#85" id="85">85</a>
+<a href="#86" id="86">86</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -140,6 +141,7 @@
#[macro_use]
</span><span class="kw">extern crate </span>derive_builder;
+<span class="kw">extern crate </span>core;
<span class="kw">mod </span>error;
<span class="kw">pub use </span>error::{Error, ErrorKind, <span class="prelude-ty">Result</span>};
diff --git a/api/src/iceberg/scan.rs.html b/api/src/iceberg/scan.rs.html
index da8f4588..454cd85a 100644
--- a/api/src/iceberg/scan.rs.html
+++ b/api/src/iceberg/scan.rs.html
@@ -1819,6 +1819,33 @@
<a href="#1819" id="1819">1819</a>
<a href="#1820" id="1820">1820</a>
<a href="#1821" id="1821">1821</a>
+<a href="#1822" id="1822">1822</a>
+<a href="#1823" id="1823">1823</a>
+<a href="#1824" id="1824">1824</a>
+<a href="#1825" id="1825">1825</a>
+<a href="#1826" id="1826">1826</a>
+<a href="#1827" id="1827">1827</a>
+<a href="#1828" id="1828">1828</a>
+<a href="#1829" id="1829">1829</a>
+<a href="#1830" id="1830">1830</a>
+<a href="#1831" id="1831">1831</a>
+<a href="#1832" id="1832">1832</a>
+<a href="#1833" id="1833">1833</a>
+<a href="#1834" id="1834">1834</a>
+<a href="#1835" id="1835">1835</a>
+<a href="#1836" id="1836">1836</a>
+<a href="#1837" id="1837">1837</a>
+<a href="#1838" id="1838">1838</a>
+<a href="#1839" id="1839">1839</a>
+<a href="#1840" id="1840">1840</a>
+<a href="#1841" id="1841">1841</a>
+<a href="#1842" id="1842">1842</a>
+<a href="#1843" id="1843">1843</a>
+<a href="#1844" id="1844">1844</a>
+<a href="#1845" id="1845">1845</a>
+<a href="#1846" id="1846">1846</a>
+<a href="#1847" id="1847">1847</a>
+<a href="#1848" id="1848">1848</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -2727,6 +2754,33 @@
</span><span class="kw">pub </span>predicate: <span class="prelude-ty">Option</span><BoundPredicate>,
}
+<span class="kw">impl </span>FileScanTask {
+ <span class="doccomment">/// Returns the data file path of this file scan task.
+ </span><span class="kw">pub fn </span>data_file_path(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span>str {
+ <span class="kw-2">&</span><span class="self">self</span>.data_file_path
+ }
+
+ <span class="doccomment">/// Returns the project field id of this file scan task.
+ </span><span class="kw">pub fn </span>project_field_ids(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span>[i32] {
+ <span class="kw-2">&</span><span class="self">self</span>.project_field_ids
+ }
+
+ <span class="doccomment">/// Returns the predicate of this file scan task.
+ </span><span class="kw">pub fn </span>predicate(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="prelude-ty">Option</span><<span class="kw-2">&</span>BoundPredicate> {
+ <span class="self">self</span>.predicate.as_ref()
+ }
+
+ <span class="doccomment">/// Returns the schema of this file scan task as a reference
+ </span><span class="kw">pub fn </span>schema(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="kw-2">&</span>Schema {
+ <span class="kw-2">&</span><span class="self">self</span>.schema
+ }
+
+ <span class="doccomment">/// Returns the schema of this file scan task as a SchemaRef
+ </span><span class="kw">pub fn </span>schema_ref(<span class="kw-2">&</span><span class="self">self</span>) -> SchemaRef {
+ <span class="self">self</span>.schema.clone()
+ }
+}
+
<span class="attr">#[cfg(test)]
</span><span class="kw">mod </span>tests {
<span class="kw">use </span>std::collections::HashMap;
diff --git a/api/trait.impl/core/convert/trait.From.js b/api/trait.impl/core/convert/trait.From.js
index 8598bfcf..cfa6eb66 100644
--- a/api/trait.impl/core/convert/trait.From.js
+++ b/api/trait.impl/core/convert/trait.From.js
@@ -1,3 +1,3 @@
(function() {var implementors = {
-"iceberg":[["impl <a class=\"trait\" href=\"https://doc.rust-lang.org/nightly/core/convert/trait.From.html\" title=\"trait core::convert::From\">From</a><<a class=\"enum\" href=\"https://docs.rs/url/2.5.2/url/parser/enum.ParseError.html\" title=\"enum url::parser::ParseError\">ParseError</a>> for <a class=\"struct\" href=\"iceberg/struct.Error.html\" title=\"struct iceberg::Error\">Error</a>"],["impl <a class=\"trait\" href=\"https://doc.rust-lang.org/nightly/core/convert/trait.Fro [...]
+"iceberg":[["impl <a class=\"trait\" href=\"https://doc.rust-lang.org/nightly/core/convert/trait.From.html\" title=\"trait core::convert::From\">From</a><<a class=\"enum\" href=\"https://docs.rs/url/2.5.2/url/parser/enum.ParseError.html\" title=\"enum url::parser::ParseError\">ParseError</a>> for <a class=\"struct\" href=\"iceberg/struct.Error.html\" title=\"struct iceberg::Error\">Error</a>"],["impl <a class=\"trait\" href=\"https://doc.rust-lang.org/nightly/core/convert/trait.Fro [...]
};if (window.register_implementors) {window.register_implementors(implementors);} else {window.pending_implementors = implementors;}})()
\ No newline at end of file