You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2024/12/16 06:29:14 UTC
(iceberg-rust) branch gh-pages updated: deploy: 97f8a792b66afbce805b1c5a4733d16ce88e602a
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/gh-pages by this push:
new f86309ec deploy: 97f8a792b66afbce805b1c5a4733d16ce88e602a
f86309ec is described below
commit f86309ecf0b4126fefa4bdf715d9327f77f3858d
Author: Xuanwo <Xu...@users.noreply.github.com>
AuthorDate: Mon Dec 16 06:29:10 2024 +0000
deploy: 97f8a792b66afbce805b1c5a4733d16ce88e602a
---
api/help.html | 2 +-
.../writer/base_writer/data_file_writer/index.html | 2 +-
api/settings.html | 2 +-
.../writer/base_writer/data_file_writer.rs.html | 236 ++++++++++++++++++++-
4 files changed, 230 insertions(+), 12 deletions(-)
diff --git a/api/help.html b/api/help.html
index b3855f23..886bccc7 100644
--- a/api/help.html
+++ b/api/help.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Documentation for Rustdoc"><title>Help</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-5 [...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Documentation for Rustdoc"><title>Help</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-5 [...]
\ No newline at end of file
diff --git a/api/iceberg/writer/base_writer/data_file_writer/index.html b/api/iceberg/writer/base_writer/data_file_writer/index.html
index 85d41995..65a97f7f 100644
--- a/api/iceberg/writer/base_writer/data_file_writer/index.html
+++ b/api/iceberg/writer/base_writer/data_file_writer/index.html
@@ -1,2 +1,2 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="This module provide `DataFileWriter`."><title>iceberg::writer::base_writer::data_file_writer - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,F [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="This module provide `DataFileWriter`."><title>iceberg::writer::base_writer::data_file_writer - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,F [...]
</div></details><h2 id="structs" class="section-header">Structs<a href="#structs" class="anchor">ยง</a></h2><ul class="item-table"><li><div class="item-name"><a class="struct" href="struct.DataFileWriter.html" title="struct iceberg::writer::base_writer::data_file_writer::DataFileWriter">DataFileWriter</a></div><div class="desc docblock-short">A writer write data is within one spec/partition.</div></li><li><div class="item-name"><a class="struct" href="struct.DataFileWriterBuilder.html" ti [...]
\ No newline at end of file
diff --git a/api/settings.html b/api/settings.html
index 0b4d7eab..430a9162 100644
--- a/api/settings.html
+++ b/api/settings.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Settings of Rustdoc"><title>Settings</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-562 [...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Settings of Rustdoc"><title>Settings</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regular-562 [...]
\ No newline at end of file
diff --git a/api/src/iceberg/writer/base_writer/data_file_writer.rs.html b/api/src/iceberg/writer/base_writer/data_file_writer.rs.html
index 9dce8a70..c49c23e2 100644
--- a/api/src/iceberg/writer/base_writer/data_file_writer.rs.html
+++ b/api/src/iceberg/writer/base_writer/data_file_writer.rs.html
@@ -139,6 +139,115 @@
<a href="#139" id="139">139</a>
<a href="#140" id="140">140</a>
<a href="#141" id="141">141</a>
+<a href="#142" id="142">142</a>
+<a href="#143" id="143">143</a>
+<a href="#144" id="144">144</a>
+<a href="#145" id="145">145</a>
+<a href="#146" id="146">146</a>
+<a href="#147" id="147">147</a>
+<a href="#148" id="148">148</a>
+<a href="#149" id="149">149</a>
+<a href="#150" id="150">150</a>
+<a href="#151" id="151">151</a>
+<a href="#152" id="152">152</a>
+<a href="#153" id="153">153</a>
+<a href="#154" id="154">154</a>
+<a href="#155" id="155">155</a>
+<a href="#156" id="156">156</a>
+<a href="#157" id="157">157</a>
+<a href="#158" id="158">158</a>
+<a href="#159" id="159">159</a>
+<a href="#160" id="160">160</a>
+<a href="#161" id="161">161</a>
+<a href="#162" id="162">162</a>
+<a href="#163" id="163">163</a>
+<a href="#164" id="164">164</a>
+<a href="#165" id="165">165</a>
+<a href="#166" id="166">166</a>
+<a href="#167" id="167">167</a>
+<a href="#168" id="168">168</a>
+<a href="#169" id="169">169</a>
+<a href="#170" id="170">170</a>
+<a href="#171" id="171">171</a>
+<a href="#172" id="172">172</a>
+<a href="#173" id="173">173</a>
+<a href="#174" id="174">174</a>
+<a href="#175" id="175">175</a>
+<a href="#176" id="176">176</a>
+<a href="#177" id="177">177</a>
+<a href="#178" id="178">178</a>
+<a href="#179" id="179">179</a>
+<a href="#180" id="180">180</a>
+<a href="#181" id="181">181</a>
+<a href="#182" id="182">182</a>
+<a href="#183" id="183">183</a>
+<a href="#184" id="184">184</a>
+<a href="#185" id="185">185</a>
+<a href="#186" id="186">186</a>
+<a href="#187" id="187">187</a>
+<a href="#188" id="188">188</a>
+<a href="#189" id="189">189</a>
+<a href="#190" id="190">190</a>
+<a href="#191" id="191">191</a>
+<a href="#192" id="192">192</a>
+<a href="#193" id="193">193</a>
+<a href="#194" id="194">194</a>
+<a href="#195" id="195">195</a>
+<a href="#196" id="196">196</a>
+<a href="#197" id="197">197</a>
+<a href="#198" id="198">198</a>
+<a href="#199" id="199">199</a>
+<a href="#200" id="200">200</a>
+<a href="#201" id="201">201</a>
+<a href="#202" id="202">202</a>
+<a href="#203" id="203">203</a>
+<a href="#204" id="204">204</a>
+<a href="#205" id="205">205</a>
+<a href="#206" id="206">206</a>
+<a href="#207" id="207">207</a>
+<a href="#208" id="208">208</a>
+<a href="#209" id="209">209</a>
+<a href="#210" id="210">210</a>
+<a href="#211" id="211">211</a>
+<a href="#212" id="212">212</a>
+<a href="#213" id="213">213</a>
+<a href="#214" id="214">214</a>
+<a href="#215" id="215">215</a>
+<a href="#216" id="216">216</a>
+<a href="#217" id="217">217</a>
+<a href="#218" id="218">218</a>
+<a href="#219" id="219">219</a>
+<a href="#220" id="220">220</a>
+<a href="#221" id="221">221</a>
+<a href="#222" id="222">222</a>
+<a href="#223" id="223">223</a>
+<a href="#224" id="224">224</a>
+<a href="#225" id="225">225</a>
+<a href="#226" id="226">226</a>
+<a href="#227" id="227">227</a>
+<a href="#228" id="228">228</a>
+<a href="#229" id="229">229</a>
+<a href="#230" id="230">230</a>
+<a href="#231" id="231">231</a>
+<a href="#232" id="232">232</a>
+<a href="#233" id="233">233</a>
+<a href="#234" id="234">234</a>
+<a href="#235" id="235">235</a>
+<a href="#236" id="236">236</a>
+<a href="#237" id="237">237</a>
+<a href="#238" id="238">238</a>
+<a href="#239" id="239">239</a>
+<a href="#240" id="240">240</a>
+<a href="#241" id="241">241</a>
+<a href="#242" id="242">242</a>
+<a href="#243" id="243">243</a>
+<a href="#244" id="244">244</a>
+<a href="#245" id="245">245</a>
+<a href="#246" id="246">246</a>
+<a href="#247" id="247">247</a>
+<a href="#248" id="248">248</a>
+<a href="#249" id="249">249</a>
+<a href="#250" id="250">250</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -241,16 +350,21 @@
</span><span class="kw">mod </span>test {
<span class="kw">use </span>std::sync::Arc;
+ <span class="kw">use </span>arrow_array::{Int32Array, StringArray};
+ <span class="kw">use </span>arrow_schema::{DataType, Field};
+ <span class="kw">use </span>parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
<span class="kw">use </span>parquet::file::properties::WriterProperties;
<span class="kw">use </span>tempfile::TempDir;
<span class="kw">use </span><span class="kw">crate</span>::io::FileIOBuilder;
- <span class="kw">use </span><span class="kw">crate</span>::spec::{DataContentType, DataFileFormat, Schema, Struct};
+ <span class="kw">use </span><span class="kw">crate</span>::spec::{
+ DataContentType, DataFileFormat, Literal, NestedField, PrimitiveType, Schema, Struct, Type,
+ };
<span class="kw">use </span><span class="kw">crate</span>::writer::base_writer::data_file_writer::DataFileWriterBuilder;
<span class="kw">use </span><span class="kw">crate</span>::writer::file_writer::location_generator::test::MockLocationGenerator;
<span class="kw">use </span><span class="kw">crate</span>::writer::file_writer::location_generator::DefaultFileNameGenerator;
<span class="kw">use </span><span class="kw">crate</span>::writer::file_writer::ParquetWriterBuilder;
- <span class="kw">use </span><span class="kw">crate</span>::writer::{IcebergWriter, IcebergWriterBuilder};
+ <span class="kw">use </span><span class="kw">crate</span>::writer::{IcebergWriter, IcebergWriterBuilder, RecordBatch};
<span class="kw">use </span><span class="kw">crate</span>::Result;
<span class="attr">#[tokio::test]
@@ -262,20 +376,124 @@
<span class="kw">let </span>file_name_gen =
DefaultFileNameGenerator::new(<span class="string">"test"</span>.to_string(), <span class="prelude-val">None</span>, DataFileFormat::Parquet);
+ <span class="kw">let </span>schema = Schema::builder()
+ .with_schema_id(<span class="number">3</span>)
+ .with_fields(<span class="macro">vec!</span>[
+ NestedField::required(<span class="number">3</span>, <span class="string">"foo"</span>, Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::required(<span class="number">4</span>, <span class="string">"bar"</span>, Type::Primitive(PrimitiveType::String)).into(),
+ ])
+ .build()<span class="question-mark">?</span>;
+
<span class="kw">let </span>pw = ParquetWriterBuilder::new(
WriterProperties::builder().build(),
- Arc::new(Schema::builder().build().unwrap()),
+ Arc::new(schema),
+ file_io.clone(),
+ location_gen,
+ file_name_gen,
+ );
+
+ <span class="kw">let </span><span class="kw-2">mut </span>data_file_writer = DataFileWriterBuilder::new(pw, <span class="prelude-val">None</span>).build().<span class="kw">await</span>.unwrap();
+
+ <span class="kw">let </span>data_files = data_file_writer.close().<span class="kw">await</span>.unwrap();
+ <span class="macro">assert_eq!</span>(data_files.len(), <span class="number">1</span>);
+
+ <span class="kw">let </span>data_file = <span class="kw-2">&</span>data_files[<span class="number">0</span>];
+ <span class="macro">assert_eq!</span>(data_file.file_format, DataFileFormat::Parquet);
+ <span class="macro">assert_eq!</span>(data_file.content, DataContentType::Data);
+ <span class="macro">assert_eq!</span>(data_file.partition, Struct::empty());
+
+ <span class="kw">let </span>input_file = file_io.new_input(data_file.file_path.clone())<span class="question-mark">?</span>;
+ <span class="kw">let </span>input_content = input_file.read().<span class="kw">await</span><span class="question-mark">?</span>;
+
+ <span class="kw">let </span>parquet_reader =
+ ArrowReaderMetadata::load(<span class="kw-2">&</span>input_content, ArrowReaderOptions::default())
+ .expect(<span class="string">"Failed to load Parquet metadata"</span>);
+
+ <span class="kw">let </span>field_ids: Vec<i32> = parquet_reader
+ .parquet_schema()
+ .columns()
+ .iter()
+ .map(|col| col.self_type().get_basic_info().id())
+ .collect();
+
+ <span class="macro">assert_eq!</span>(field_ids, <span class="macro">vec!</span>[<span class="number">3</span>, <span class="number">4</span>]);
+ <span class="prelude-val">Ok</span>(())
+ }
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_parquet_writer_with_partition() -> <span class="prelude-ty">Result</span><()> {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+ <span class="kw">let </span>file_io = FileIOBuilder::new_fs_io().build().unwrap();
+ <span class="kw">let </span>location_gen =
+ MockLocationGenerator::new(temp_dir.path().to_str().unwrap().to_string());
+ <span class="kw">let </span>file_name_gen = DefaultFileNameGenerator::new(
+ <span class="string">"test_partitioned"</span>.to_string(),
+ <span class="prelude-val">None</span>,
+ DataFileFormat::Parquet,
+ );
+
+ <span class="kw">let </span>schema = Schema::builder()
+ .with_schema_id(<span class="number">5</span>)
+ .with_fields(<span class="macro">vec!</span>[
+ NestedField::required(<span class="number">5</span>, <span class="string">"id"</span>, Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::required(<span class="number">6</span>, <span class="string">"name"</span>, Type::Primitive(PrimitiveType::String)).into(),
+ ])
+ .build()<span class="question-mark">?</span>;
+
+ <span class="kw">let </span>partition_value = Struct::from_iter([<span class="prelude-val">Some</span>(Literal::int(<span class="number">1</span>))]);
+
+ <span class="kw">let </span>parquet_writer_builder = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ Arc::new(schema.clone()),
file_io.clone(),
location_gen,
file_name_gen,
);
- <span class="kw">let </span><span class="kw-2">mut </span>data_file_writer = DataFileWriterBuilder::new(pw, <span class="prelude-val">None</span>).build().<span class="kw">await</span><span class="question-mark">?</span>;
- <span class="kw">let </span>data_file = data_file_writer.close().<span class="kw">await</span>.unwrap();
- <span class="macro">assert_eq!</span>(data_file.len(), <span class="number">1</span>);
- <span class="macro">assert_eq!</span>(data_file[<span class="number">0</span>].file_format, DataFileFormat::Parquet);
- <span class="macro">assert_eq!</span>(data_file[<span class="number">0</span>].content, DataContentType::Data);
- <span class="macro">assert_eq!</span>(data_file[<span class="number">0</span>].partition, Struct::empty());
+ <span class="kw">let </span><span class="kw-2">mut </span>data_file_writer =
+ DataFileWriterBuilder::new(parquet_writer_builder, <span class="prelude-val">Some</span>(partition_value.clone()))
+ .build()
+ .<span class="kw">await</span><span class="question-mark">?</span>;
+
+ <span class="kw">let </span>arrow_schema = arrow_schema::Schema::new(<span class="macro">vec!</span>[
+ Field::new(<span class="string">"id"</span>, DataType::Int32, <span class="bool-val">false</span>),
+ Field::new(<span class="string">"name"</span>, DataType::Utf8, <span class="bool-val">false</span>),
+ ]);
+ <span class="kw">let </span>batch = RecordBatch::try_new(Arc::new(arrow_schema.clone()), <span class="macro">vec!</span>[
+ Arc::new(Int32Array::from(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])),
+ Arc::new(StringArray::from(<span class="macro">vec!</span>[<span class="string">"Alice"</span>, <span class="string">"Bob"</span>, <span class="string">"Charlie"</span>])),
+ ])<span class="question-mark">?</span>;
+ data_file_writer.write(batch).<span class="kw">await</span><span class="question-mark">?</span>;
+
+ <span class="kw">let </span>data_files = data_file_writer.close().<span class="kw">await</span>.unwrap();
+ <span class="macro">assert_eq!</span>(data_files.len(), <span class="number">1</span>);
+
+ <span class="kw">let </span>data_file = <span class="kw-2">&</span>data_files[<span class="number">0</span>];
+ <span class="macro">assert_eq!</span>(data_file.file_format, DataFileFormat::Parquet);
+ <span class="macro">assert_eq!</span>(data_file.content, DataContentType::Data);
+ <span class="macro">assert_eq!</span>(data_file.partition, partition_value);
+
+ <span class="kw">let </span>input_file = file_io.new_input(data_file.file_path.clone())<span class="question-mark">?</span>;
+ <span class="kw">let </span>input_content = input_file.read().<span class="kw">await</span><span class="question-mark">?</span>;
+
+ <span class="kw">let </span>parquet_reader =
+ ArrowReaderMetadata::load(<span class="kw-2">&</span>input_content, ArrowReaderOptions::default())<span class="question-mark">?</span>;
+
+ <span class="kw">let </span>field_ids: Vec<i32> = parquet_reader
+ .parquet_schema()
+ .columns()
+ .iter()
+ .map(|col| col.self_type().get_basic_info().id())
+ .collect();
+ <span class="macro">assert_eq!</span>(field_ids, <span class="macro">vec!</span>[<span class="number">5</span>, <span class="number">6</span>]);
+
+ <span class="kw">let </span>field_names: Vec<<span class="kw-2">&</span>str> = parquet_reader
+ .parquet_schema()
+ .columns()
+ .iter()
+ .map(|col| col.name())
+ .collect();
+ <span class="macro">assert_eq!</span>(field_names, <span class="macro">vec!</span>[<span class="string">"id"</span>, <span class="string">"name"</span>]);
<span class="prelude-val">Ok</span>(())
}