You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2025/04/01 13:19:20 UTC
(iceberg-rust) branch gh-pages updated: deploy: 76e87a820439b6561439a0861119503646172f44
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/gh-pages by this push:
new 861cfcf2 deploy: 76e87a820439b6561439a0861119503646172f44
861cfcf2 is described below
commit 861cfcf2d7a9d02153889182bb7acf674fd14b4c
Author: liurenjie1024 <li...@users.noreply.github.com>
AuthorDate: Tue Apr 1 13:19:15 2025 +0000
deploy: 76e87a820439b6561439a0861119503646172f44
---
api/src-files.js | 2 +-
api/src/iceberg/puffin/mod.rs.html | 2 +
api/src/iceberg/puffin/writer.rs.html | 677 ++++++++++++++++++++++++++++++++++
3 files changed, 680 insertions(+), 1 deletion(-)
diff --git a/api/src-files.js b/api/src-files.js
index 63b9fa2b..2431fd62 100644
--- a/api/src-files.js
+++ b/api/src-files.js
@@ -1,5 +1,5 @@
var srcIndex = new Map(JSON.parse('[\
-["iceberg",["",[["arrow",[],["delete_file_manager.rs","mod.rs","nan_val_cnt_visitor.rs","reader.rs","record_batch_projector.rs","record_batch_transformer.rs","schema.rs","value.rs"]],["avro",[],["mod.rs","schema.rs"]],["catalog",[],["mod.rs"]],["expr",[["visitors",[],["bound_predicate_visitor.rs","expression_evaluator.rs","inclusive_metrics_evaluator.rs","inclusive_projection.rs","manifest_evaluator.rs","mod.rs","page_index_evaluator.rs","row_group_metrics_evaluator.rs","strict_metrics_e [...]
+["iceberg",["",[["arrow",[],["delete_file_manager.rs","mod.rs","nan_val_cnt_visitor.rs","reader.rs","record_batch_projector.rs","record_batch_transformer.rs","schema.rs","value.rs"]],["avro",[],["mod.rs","schema.rs"]],["catalog",[],["mod.rs"]],["expr",[["visitors",[],["bound_predicate_visitor.rs","expression_evaluator.rs","inclusive_metrics_evaluator.rs","inclusive_projection.rs","manifest_evaluator.rs","mod.rs","page_index_evaluator.rs","row_group_metrics_evaluator.rs","strict_metrics_e [...]
["iceberg_catalog_glue",["",[],["catalog.rs","error.rs","lib.rs","schema.rs","utils.rs"]]],\
["iceberg_catalog_hms",["",[],["catalog.rs","error.rs","lib.rs","schema.rs","utils.rs"]]],\
["iceberg_catalog_memory",["",[],["catalog.rs","lib.rs","namespace_state.rs"]]],\
diff --git a/api/src/iceberg/puffin/mod.rs.html b/api/src/iceberg/puffin/mod.rs.html
index 52107504..fbf62302 100644
--- a/api/src/iceberg/puffin/mod.rs.html
+++ b/api/src/iceberg/puffin/mod.rs.html
@@ -29,6 +29,7 @@
<a href="#29" id="29">29</a>
<a href="#30" id="30">30</a>
<a href="#31" id="31">31</a>
+<a href="#32" id="32">32</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -57,6 +58,7 @@
<span class="kw">mod </span>metadata;
<span class="attr">#[cfg(feature = <span class="string">"tokio"</span>)]
</span><span class="kw">mod </span>reader;
+<span class="kw">mod </span>writer;
<span class="attr">#[cfg(test)]
</span><span class="kw">mod </span>test_utils;
diff --git a/api/src/iceberg/puffin/writer.rs.html b/api/src/iceberg/puffin/writer.rs.html
new file mode 100644
index 00000000..3f6d89a7
--- /dev/null
+++ b/api/src/iceberg/puffin/writer.rs.html
@@ -0,0 +1,677 @@
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `crates/iceberg/src/puffin/writer.rs`."><title>writer.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Me [...]
+<a href="#2" id="2">2</a>
+<a href="#3" id="3">3</a>
+<a href="#4" id="4">4</a>
+<a href="#5" id="5">5</a>
+<a href="#6" id="6">6</a>
+<a href="#7" id="7">7</a>
+<a href="#8" id="8">8</a>
+<a href="#9" id="9">9</a>
+<a href="#10" id="10">10</a>
+<a href="#11" id="11">11</a>
+<a href="#12" id="12">12</a>
+<a href="#13" id="13">13</a>
+<a href="#14" id="14">14</a>
+<a href="#15" id="15">15</a>
+<a href="#16" id="16">16</a>
+<a href="#17" id="17">17</a>
+<a href="#18" id="18">18</a>
+<a href="#19" id="19">19</a>
+<a href="#20" id="20">20</a>
+<a href="#21" id="21">21</a>
+<a href="#22" id="22">22</a>
+<a href="#23" id="23">23</a>
+<a href="#24" id="24">24</a>
+<a href="#25" id="25">25</a>
+<a href="#26" id="26">26</a>
+<a href="#27" id="27">27</a>
+<a href="#28" id="28">28</a>
+<a href="#29" id="29">29</a>
+<a href="#30" id="30">30</a>
+<a href="#31" id="31">31</a>
+<a href="#32" id="32">32</a>
+<a href="#33" id="33">33</a>
+<a href="#34" id="34">34</a>
+<a href="#35" id="35">35</a>
+<a href="#36" id="36">36</a>
+<a href="#37" id="37">37</a>
+<a href="#38" id="38">38</a>
+<a href="#39" id="39">39</a>
+<a href="#40" id="40">40</a>
+<a href="#41" id="41">41</a>
+<a href="#42" id="42">42</a>
+<a href="#43" id="43">43</a>
+<a href="#44" id="44">44</a>
+<a href="#45" id="45">45</a>
+<a href="#46" id="46">46</a>
+<a href="#47" id="47">47</a>
+<a href="#48" id="48">48</a>
+<a href="#49" id="49">49</a>
+<a href="#50" id="50">50</a>
+<a href="#51" id="51">51</a>
+<a href="#52" id="52">52</a>
+<a href="#53" id="53">53</a>
+<a href="#54" id="54">54</a>
+<a href="#55" id="55">55</a>
+<a href="#56" id="56">56</a>
+<a href="#57" id="57">57</a>
+<a href="#58" id="58">58</a>
+<a href="#59" id="59">59</a>
+<a href="#60" id="60">60</a>
+<a href="#61" id="61">61</a>
+<a href="#62" id="62">62</a>
+<a href="#63" id="63">63</a>
+<a href="#64" id="64">64</a>
+<a href="#65" id="65">65</a>
+<a href="#66" id="66">66</a>
+<a href="#67" id="67">67</a>
+<a href="#68" id="68">68</a>
+<a href="#69" id="69">69</a>
+<a href="#70" id="70">70</a>
+<a href="#71" id="71">71</a>
+<a href="#72" id="72">72</a>
+<a href="#73" id="73">73</a>
+<a href="#74" id="74">74</a>
+<a href="#75" id="75">75</a>
+<a href="#76" id="76">76</a>
+<a href="#77" id="77">77</a>
+<a href="#78" id="78">78</a>
+<a href="#79" id="79">79</a>
+<a href="#80" id="80">80</a>
+<a href="#81" id="81">81</a>
+<a href="#82" id="82">82</a>
+<a href="#83" id="83">83</a>
+<a href="#84" id="84">84</a>
+<a href="#85" id="85">85</a>
+<a href="#86" id="86">86</a>
+<a href="#87" id="87">87</a>
+<a href="#88" id="88">88</a>
+<a href="#89" id="89">89</a>
+<a href="#90" id="90">90</a>
+<a href="#91" id="91">91</a>
+<a href="#92" id="92">92</a>
+<a href="#93" id="93">93</a>
+<a href="#94" id="94">94</a>
+<a href="#95" id="95">95</a>
+<a href="#96" id="96">96</a>
+<a href="#97" id="97">97</a>
+<a href="#98" id="98">98</a>
+<a href="#99" id="99">99</a>
+<a href="#100" id="100">100</a>
+<a href="#101" id="101">101</a>
+<a href="#102" id="102">102</a>
+<a href="#103" id="103">103</a>
+<a href="#104" id="104">104</a>
+<a href="#105" id="105">105</a>
+<a href="#106" id="106">106</a>
+<a href="#107" id="107">107</a>
+<a href="#108" id="108">108</a>
+<a href="#109" id="109">109</a>
+<a href="#110" id="110">110</a>
+<a href="#111" id="111">111</a>
+<a href="#112" id="112">112</a>
+<a href="#113" id="113">113</a>
+<a href="#114" id="114">114</a>
+<a href="#115" id="115">115</a>
+<a href="#116" id="116">116</a>
+<a href="#117" id="117">117</a>
+<a href="#118" id="118">118</a>
+<a href="#119" id="119">119</a>
+<a href="#120" id="120">120</a>
+<a href="#121" id="121">121</a>
+<a href="#122" id="122">122</a>
+<a href="#123" id="123">123</a>
+<a href="#124" id="124">124</a>
+<a href="#125" id="125">125</a>
+<a href="#126" id="126">126</a>
+<a href="#127" id="127">127</a>
+<a href="#128" id="128">128</a>
+<a href="#129" id="129">129</a>
+<a href="#130" id="130">130</a>
+<a href="#131" id="131">131</a>
+<a href="#132" id="132">132</a>
+<a href="#133" id="133">133</a>
+<a href="#134" id="134">134</a>
+<a href="#135" id="135">135</a>
+<a href="#136" id="136">136</a>
+<a href="#137" id="137">137</a>
+<a href="#138" id="138">138</a>
+<a href="#139" id="139">139</a>
+<a href="#140" id="140">140</a>
+<a href="#141" id="141">141</a>
+<a href="#142" id="142">142</a>
+<a href="#143" id="143">143</a>
+<a href="#144" id="144">144</a>
+<a href="#145" id="145">145</a>
+<a href="#146" id="146">146</a>
+<a href="#147" id="147">147</a>
+<a href="#148" id="148">148</a>
+<a href="#149" id="149">149</a>
+<a href="#150" id="150">150</a>
+<a href="#151" id="151">151</a>
+<a href="#152" id="152">152</a>
+<a href="#153" id="153">153</a>
+<a href="#154" id="154">154</a>
+<a href="#155" id="155">155</a>
+<a href="#156" id="156">156</a>
+<a href="#157" id="157">157</a>
+<a href="#158" id="158">158</a>
+<a href="#159" id="159">159</a>
+<a href="#160" id="160">160</a>
+<a href="#161" id="161">161</a>
+<a href="#162" id="162">162</a>
+<a href="#163" id="163">163</a>
+<a href="#164" id="164">164</a>
+<a href="#165" id="165">165</a>
+<a href="#166" id="166">166</a>
+<a href="#167" id="167">167</a>
+<a href="#168" id="168">168</a>
+<a href="#169" id="169">169</a>
+<a href="#170" id="170">170</a>
+<a href="#171" id="171">171</a>
+<a href="#172" id="172">172</a>
+<a href="#173" id="173">173</a>
+<a href="#174" id="174">174</a>
+<a href="#175" id="175">175</a>
+<a href="#176" id="176">176</a>
+<a href="#177" id="177">177</a>
+<a href="#178" id="178">178</a>
+<a href="#179" id="179">179</a>
+<a href="#180" id="180">180</a>
+<a href="#181" id="181">181</a>
+<a href="#182" id="182">182</a>
+<a href="#183" id="183">183</a>
+<a href="#184" id="184">184</a>
+<a href="#185" id="185">185</a>
+<a href="#186" id="186">186</a>
+<a href="#187" id="187">187</a>
+<a href="#188" id="188">188</a>
+<a href="#189" id="189">189</a>
+<a href="#190" id="190">190</a>
+<a href="#191" id="191">191</a>
+<a href="#192" id="192">192</a>
+<a href="#193" id="193">193</a>
+<a href="#194" id="194">194</a>
+<a href="#195" id="195">195</a>
+<a href="#196" id="196">196</a>
+<a href="#197" id="197">197</a>
+<a href="#198" id="198">198</a>
+<a href="#199" id="199">199</a>
+<a href="#200" id="200">200</a>
+<a href="#201" id="201">201</a>
+<a href="#202" id="202">202</a>
+<a href="#203" id="203">203</a>
+<a href="#204" id="204">204</a>
+<a href="#205" id="205">205</a>
+<a href="#206" id="206">206</a>
+<a href="#207" id="207">207</a>
+<a href="#208" id="208">208</a>
+<a href="#209" id="209">209</a>
+<a href="#210" id="210">210</a>
+<a href="#211" id="211">211</a>
+<a href="#212" id="212">212</a>
+<a href="#213" id="213">213</a>
+<a href="#214" id="214">214</a>
+<a href="#215" id="215">215</a>
+<a href="#216" id="216">216</a>
+<a href="#217" id="217">217</a>
+<a href="#218" id="218">218</a>
+<a href="#219" id="219">219</a>
+<a href="#220" id="220">220</a>
+<a href="#221" id="221">221</a>
+<a href="#222" id="222">222</a>
+<a href="#223" id="223">223</a>
+<a href="#224" id="224">224</a>
+<a href="#225" id="225">225</a>
+<a href="#226" id="226">226</a>
+<a href="#227" id="227">227</a>
+<a href="#228" id="228">228</a>
+<a href="#229" id="229">229</a>
+<a href="#230" id="230">230</a>
+<a href="#231" id="231">231</a>
+<a href="#232" id="232">232</a>
+<a href="#233" id="233">233</a>
+<a href="#234" id="234">234</a>
+<a href="#235" id="235">235</a>
+<a href="#236" id="236">236</a>
+<a href="#237" id="237">237</a>
+<a href="#238" id="238">238</a>
+<a href="#239" id="239">239</a>
+<a href="#240" id="240">240</a>
+<a href="#241" id="241">241</a>
+<a href="#242" id="242">242</a>
+<a href="#243" id="243">243</a>
+<a href="#244" id="244">244</a>
+<a href="#245" id="245">245</a>
+<a href="#246" id="246">246</a>
+<a href="#247" id="247">247</a>
+<a href="#248" id="248">248</a>
+<a href="#249" id="249">249</a>
+<a href="#250" id="250">250</a>
+<a href="#251" id="251">251</a>
+<a href="#252" id="252">252</a>
+<a href="#253" id="253">253</a>
+<a href="#254" id="254">254</a>
+<a href="#255" id="255">255</a>
+<a href="#256" id="256">256</a>
+<a href="#257" id="257">257</a>
+<a href="#258" id="258">258</a>
+<a href="#259" id="259">259</a>
+<a href="#260" id="260">260</a>
+<a href="#261" id="261">261</a>
+<a href="#262" id="262">262</a>
+<a href="#263" id="263">263</a>
+<a href="#264" id="264">264</a>
+<a href="#265" id="265">265</a>
+<a href="#266" id="266">266</a>
+<a href="#267" id="267">267</a>
+<a href="#268" id="268">268</a>
+<a href="#269" id="269">269</a>
+<a href="#270" id="270">270</a>
+<a href="#271" id="271">271</a>
+<a href="#272" id="272">272</a>
+<a href="#273" id="273">273</a>
+<a href="#274" id="274">274</a>
+<a href="#275" id="275">275</a>
+<a href="#276" id="276">276</a>
+<a href="#277" id="277">277</a>
+<a href="#278" id="278">278</a>
+<a href="#279" id="279">279</a>
+<a href="#280" id="280">280</a>
+<a href="#281" id="281">281</a>
+<a href="#282" id="282">282</a>
+<a href="#283" id="283">283</a>
+<a href="#284" id="284">284</a>
+<a href="#285" id="285">285</a>
+<a href="#286" id="286">286</a>
+<a href="#287" id="287">287</a>
+<a href="#288" id="288">288</a>
+<a href="#289" id="289">289</a>
+<a href="#290" id="290">290</a>
+<a href="#291" id="291">291</a>
+<a href="#292" id="292">292</a>
+<a href="#293" id="293">293</a>
+<a href="#294" id="294">294</a>
+<a href="#295" id="295">295</a>
+<a href="#296" id="296">296</a>
+<a href="#297" id="297">297</a>
+<a href="#298" id="298">298</a>
+<a href="#299" id="299">299</a>
+<a href="#300" id="300">300</a>
+<a href="#301" id="301">301</a>
+<a href="#302" id="302">302</a>
+<a href="#303" id="303">303</a>
+<a href="#304" id="304">304</a>
+<a href="#305" id="305">305</a>
+<a href="#306" id="306">306</a>
+<a href="#307" id="307">307</a>
+<a href="#308" id="308">308</a>
+<a href="#309" id="309">309</a>
+<a href="#310" id="310">310</a>
+<a href="#311" id="311">311</a>
+<a href="#312" id="312">312</a>
+<a href="#313" id="313">313</a>
+<a href="#314" id="314">314</a>
+<a href="#315" id="315">315</a>
+<a href="#316" id="316">316</a>
+<a href="#317" id="317">317</a>
+<a href="#318" id="318">318</a>
+<a href="#319" id="319">319</a>
+<a href="#320" id="320">320</a>
+<a href="#321" id="321">321</a>
+<a href="#322" id="322">322</a>
+<a href="#323" id="323">323</a>
+<a href="#324" id="324">324</a>
+<a href="#325" id="325">325</a>
+<a href="#326" id="326">326</a>
+<a href="#327" id="327">327</a>
+<a href="#328" id="328">328</a>
+<a href="#329" id="329">329</a>
+<a href="#330" id="330">330</a>
+<a href="#331" id="331">331</a>
+<a href="#332" id="332">332</a>
+<a href="#333" id="333">333</a>
+<a href="#334" id="334">334</a>
+<a href="#335" id="335">335</a>
+<a href="#336" id="336">336</a>
+<a href="#337" id="337">337</a>
+<a href="#338" id="338">338</a>
+</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+</span><span class="kw">use </span>std::collections::{HashMap, HashSet};
+
+<span class="kw">use </span>bytes::Bytes;
+
+<span class="kw">use </span><span class="kw">crate</span>::io::{FileWrite, OutputFile};
+<span class="kw">use </span><span class="kw">crate</span>::puffin::blob::Blob;
+<span class="kw">use </span><span class="kw">crate</span>::puffin::compression::CompressionCodec;
+<span class="kw">use </span><span class="kw">crate</span>::puffin::metadata::{BlobMetadata, FileMetadata, Flag};
+<span class="kw">use </span><span class="kw">crate</span>::Result;
+
+<span class="doccomment">/// Puffin writer
+</span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">struct </span>PuffinWriter {
+ writer: Box<<span class="kw">dyn </span>FileWrite>,
+ is_header_written: bool,
+ num_bytes_written: u64,
+ written_blobs_metadata: Vec<BlobMetadata>,
+ properties: HashMap<String, String>,
+ footer_compression_codec: CompressionCodec,
+ flags: HashSet<Flag>,
+}
+
+<span class="kw">impl </span>PuffinWriter {
+ <span class="doccomment">/// Returns a new Puffin writer
+ </span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>new(
+ output_file: <span class="kw-2">&</span>OutputFile,
+ properties: HashMap<String, String>,
+ compress_footer: bool,
+ ) -> <span class="prelude-ty">Result</span><<span class="self">Self</span>> {
+ <span class="kw">let </span><span class="kw-2">mut </span>flags = HashSet::<Flag>::new();
+ <span class="kw">let </span>footer_compression_codec = <span class="kw">if </span>compress_footer {
+ flags.insert(Flag::FooterPayloadCompressed);
+ CompressionCodec::Lz4
+ } <span class="kw">else </span>{
+ CompressionCodec::None
+ };
+
+ <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
+ writer: output_file.writer().<span class="kw">await</span><span class="question-mark">?</span>,
+ is_header_written: <span class="bool-val">false</span>,
+ num_bytes_written: <span class="number">0</span>,
+ written_blobs_metadata: Vec::new(),
+ properties,
+ footer_compression_codec,
+ flags,
+ })
+ }
+
+ <span class="doccomment">/// Adds blob to Puffin file
+ </span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>add(
+ <span class="kw-2">&mut </span><span class="self">self</span>,
+ blob: Blob,
+ compression_codec: CompressionCodec,
+ ) -> <span class="prelude-ty">Result</span><()> {
+ <span class="self">self</span>.write_header_once().<span class="kw">await</span><span class="question-mark">?</span>;
+
+ <span class="kw">let </span>offset = <span class="self">self</span>.num_bytes_written;
+ <span class="kw">let </span>compressed_bytes: Bytes = compression_codec.compress(blob.data)<span class="question-mark">?</span>.into();
+ <span class="kw">let </span>length = compressed_bytes.len().try_into()<span class="question-mark">?</span>;
+ <span class="self">self</span>.write(compressed_bytes).<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="self">self</span>.written_blobs_metadata.push(BlobMetadata {
+ r#type: blob.r#type,
+ fields: blob.fields,
+ snapshot_id: blob.snapshot_id,
+ sequence_number: blob.sequence_number,
+ offset,
+ length,
+ compression_codec,
+ properties: blob.properties,
+ });
+
+ <span class="prelude-val">Ok</span>(())
+ }
+
+ <span class="doccomment">/// Finalizes the Puffin file
+ </span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>close(<span class="kw-2">mut </span><span class="self">self</span>) -> <span class="prelude-ty">Result</span><()> {
+ <span class="self">self</span>.write_header_once().<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="self">self</span>.write_footer().<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="self">self</span>.writer.close().<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="prelude-val">Ok</span>(())
+ }
+
+ <span class="kw">async fn </span>write(<span class="kw-2">&mut </span><span class="self">self</span>, bytes: Bytes) -> <span class="prelude-ty">Result</span><()> {
+ <span class="kw">let </span>length = bytes.len();
+ <span class="self">self</span>.writer.write(bytes).<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="self">self</span>.num_bytes_written += length <span class="kw">as </span>u64;
+ <span class="prelude-val">Ok</span>(())
+ }
+
+ <span class="kw">async fn </span>write_header_once(<span class="kw-2">&mut </span><span class="self">self</span>) -> <span class="prelude-ty">Result</span><()> {
+ <span class="kw">if </span>!<span class="self">self</span>.is_header_written {
+ <span class="kw">let </span>bytes = Bytes::copy_from_slice(<span class="kw-2">&</span>FileMetadata::MAGIC);
+ <span class="self">self</span>.write(bytes).<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="self">self</span>.is_header_written = <span class="bool-val">true</span>;
+ }
+ <span class="prelude-val">Ok</span>(())
+ }
+
+ <span class="kw">fn </span>footer_payload_bytes(<span class="kw-2">&</span><span class="self">self</span>) -> <span class="prelude-ty">Result</span><Vec<u8>> {
+ <span class="kw">let </span>file_metadata = FileMetadata {
+ blobs: <span class="self">self</span>.written_blobs_metadata.clone(),
+ properties: <span class="self">self</span>.properties.clone(),
+ };
+ <span class="kw">let </span>json = serde_json::to_string::<FileMetadata>(<span class="kw-2">&</span>file_metadata)<span class="question-mark">?</span>;
+ <span class="kw">let </span>bytes = json.as_bytes();
+ <span class="self">self</span>.footer_compression_codec.compress(bytes.to_vec())
+ }
+
+ <span class="kw">fn </span>flags_bytes(<span class="kw-2">&</span><span class="self">self</span>) -> [u8; FileMetadata::FOOTER_STRUCT_FLAGS_LENGTH <span class="kw">as </span>usize] {
+ <span class="kw">let </span><span class="kw-2">mut </span>result = [<span class="number">0</span>; FileMetadata::FOOTER_STRUCT_FLAGS_LENGTH <span class="kw">as </span>usize];
+ <span class="kw">for </span>flag <span class="kw">in </span><span class="kw-2">&</span><span class="self">self</span>.flags {
+ <span class="kw">let </span>byte_idx: usize = flag.byte_idx().into();
+ result[byte_idx] |= <span class="number">0x1 </span><< flag.bit_idx();
+ }
+ result
+ }
+
+ <span class="kw">async fn </span>write_footer(<span class="kw-2">&mut </span><span class="self">self</span>) -> <span class="prelude-ty">Result</span><()> {
+ <span class="kw">let </span><span class="kw-2">mut </span>footer_payload_bytes = <span class="self">self</span>.footer_payload_bytes()<span class="question-mark">?</span>;
+ <span class="kw">let </span>footer_payload_bytes_length = u32::to_le_bytes(footer_payload_bytes.len().try_into()<span class="question-mark">?</span>);
+
+ <span class="kw">let </span><span class="kw-2">mut </span>footer_bytes = Vec::new();
+ footer_bytes.extend(<span class="kw-2">&</span>FileMetadata::MAGIC);
+ footer_bytes.append(<span class="kw-2">&mut </span>footer_payload_bytes);
+ footer_bytes.extend(footer_payload_bytes_length);
+ footer_bytes.extend(<span class="self">self</span>.flags_bytes());
+ footer_bytes.extend(<span class="kw-2">&</span>FileMetadata::MAGIC);
+
+ <span class="self">self</span>.write(footer_bytes.into()).<span class="kw">await
+ </span>}
+}
+
+<span class="attr">#[cfg(test)]
+</span><span class="kw">mod </span>tests {
+ <span class="kw">use </span>std::collections::HashMap;
+
+ <span class="kw">use </span>tempfile::TempDir;
+
+ <span class="kw">use </span><span class="kw">crate</span>::io::{FileIOBuilder, InputFile, OutputFile};
+ <span class="kw">use </span><span class="kw">crate</span>::puffin::blob::Blob;
+ <span class="kw">use </span><span class="kw">crate</span>::puffin::compression::CompressionCodec;
+ <span class="kw">use </span><span class="kw">crate</span>::puffin::metadata::FileMetadata;
+ <span class="kw">use </span><span class="kw">crate</span>::puffin::reader::PuffinReader;
+ <span class="kw">use </span><span class="kw">crate</span>::puffin::test_utils::{
+ blob_0, blob_1, empty_footer_payload, empty_footer_payload_bytes, file_properties,
+ java_empty_uncompressed_input_file, java_uncompressed_metric_input_file,
+ java_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata,
+ zstd_compressed_metric_file_metadata,
+ };
+ <span class="kw">use </span><span class="kw">crate</span>::puffin::writer::PuffinWriter;
+ <span class="kw">use </span><span class="kw">crate</span>::Result;
+
+ <span class="kw">async fn </span>write_puffin_file(
+ temp_dir: <span class="kw-2">&</span>TempDir,
+ blobs: Vec<(Blob, CompressionCodec)>,
+ properties: HashMap<String, String>,
+ ) -> <span class="prelude-ty">Result</span><OutputFile> {
+ <span class="kw">let </span>file_io = FileIOBuilder::new_fs_io().build()<span class="question-mark">?</span>;
+
+ <span class="kw">let </span>path_buf = temp_dir.path().join(<span class="string">"temp_puffin.bin"</span>);
+ <span class="kw">let </span>temp_path = path_buf.to_str().unwrap();
+ <span class="kw">let </span>output_file = file_io.new_output(temp_path)<span class="question-mark">?</span>;
+
+ <span class="kw">let </span><span class="kw-2">mut </span>writer = PuffinWriter::new(<span class="kw-2">&</span>output_file, properties, <span class="bool-val">false</span>).<span class="kw">await</span><span class="question-mark">?</span>;
+ <span class="kw">for </span>(blob, compression_codec) <span class="kw">in </span>blobs {
+ writer.add(blob, compression_codec).<span class="kw">await</span><span class="question-mark">?</span>;
+ }
+ writer.close().<span class="kw">await</span><span class="question-mark">?</span>;
+
+ <span class="prelude-val">Ok</span>(output_file)
+ }
+
+ <span class="kw">async fn </span>read_all_blobs_from_puffin_file(input_file: InputFile) -> Vec<Blob> {
+ <span class="kw">let </span>puffin_reader = PuffinReader::new(input_file);
+ <span class="kw">let </span><span class="kw-2">mut </span>blobs = Vec::new();
+ <span class="kw">let </span>blobs_metadata = puffin_reader.file_metadata().<span class="kw">await</span>.unwrap().clone().blobs;
+ <span class="kw">for </span>blob_metadata <span class="kw">in </span>blobs_metadata {
+ blobs.push(puffin_reader.blob(<span class="kw-2">&</span>blob_metadata).<span class="kw">await</span>.unwrap());
+ }
+ blobs
+ }
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_write_uncompressed_empty_file() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+
+ <span class="kw">let </span>input_file = write_puffin_file(<span class="kw-2">&</span>temp_dir, Vec::new(), HashMap::new())
+ .<span class="kw">await
+ </span>.unwrap()
+ .to_input_file();
+
+ <span class="macro">assert_eq!</span>(
+ FileMetadata::read(<span class="kw-2">&</span>input_file).<span class="kw">await</span>.unwrap(),
+ empty_footer_payload()
+ );
+
+ <span class="macro">assert_eq!</span>(
+ input_file.read().<span class="kw">await</span>.unwrap().len(),
+ FileMetadata::MAGIC_LENGTH <span class="kw">as </span>usize
+ <span class="comment">// no blobs since puffin file is empty
+ </span>+ FileMetadata::MAGIC_LENGTH <span class="kw">as </span>usize
+ + empty_footer_payload_bytes().len()
+ + FileMetadata::FOOTER_STRUCT_LENGTH <span class="kw">as </span>usize
+ )
+ }
+
+ <span class="kw">fn </span>blobs_with_compression(
+ blobs: Vec<Blob>,
+ compression_codec: CompressionCodec,
+ ) -> Vec<(Blob, CompressionCodec)> {
+ blobs
+ .into_iter()
+ .map(|blob| (blob, compression_codec))
+ .collect()
+ }
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_write_uncompressed_metric_data() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+ <span class="kw">let </span>blobs = <span class="macro">vec!</span>[blob_0(), blob_1()];
+ <span class="kw">let </span>blobs_with_compression = blobs_with_compression(blobs.clone(), CompressionCodec::None);
+
+ <span class="kw">let </span>input_file = write_puffin_file(<span class="kw-2">&</span>temp_dir, blobs_with_compression, file_properties())
+ .<span class="kw">await
+ </span>.unwrap()
+ .to_input_file();
+
+ <span class="macro">assert_eq!</span>(
+ FileMetadata::read(<span class="kw-2">&</span>input_file).<span class="kw">await</span>.unwrap(),
+ uncompressed_metric_file_metadata()
+ );
+
+ <span class="macro">assert_eq!</span>(read_all_blobs_from_puffin_file(input_file).<span class="kw">await</span>, blobs)
+ }
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_write_zstd_compressed_metric_data() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+ <span class="kw">let </span>blobs = <span class="macro">vec!</span>[blob_0(), blob_1()];
+ <span class="kw">let </span>blobs_with_compression = blobs_with_compression(blobs.clone(), CompressionCodec::Zstd);
+
+ <span class="kw">let </span>input_file = write_puffin_file(<span class="kw-2">&</span>temp_dir, blobs_with_compression, file_properties())
+ .<span class="kw">await
+ </span>.unwrap()
+ .to_input_file();
+
+ <span class="macro">assert_eq!</span>(
+ FileMetadata::read(<span class="kw-2">&</span>input_file).<span class="kw">await</span>.unwrap(),
+ zstd_compressed_metric_file_metadata()
+ );
+
+ <span class="macro">assert_eq!</span>(read_all_blobs_from_puffin_file(input_file).<span class="kw">await</span>, blobs)
+ }
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_write_lz4_compressed_metric_data() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+ <span class="kw">let </span>blobs = <span class="macro">vec!</span>[blob_0(), blob_1()];
+ <span class="kw">let </span>blobs_with_compression = blobs_with_compression(blobs.clone(), CompressionCodec::Lz4);
+
+ <span class="macro">assert_eq!</span>(
+ write_puffin_file(<span class="kw-2">&</span>temp_dir, blobs_with_compression, file_properties())
+ .<span class="kw">await
+ </span>.unwrap_err()
+ .to_string(),
+ <span class="string">"FeatureUnsupported => LZ4 compression is not supported currently"
+ </span>);
+ }
+
+ <span class="kw">async fn </span>get_file_as_byte_vec(input_file: InputFile) -> Vec<u8> {
+ input_file.read().<span class="kw">await</span>.unwrap().to_vec()
+ }
+
+ <span class="kw">async fn </span>assert_files_are_bit_identical(actual: OutputFile, expected: InputFile) {
+ <span class="kw">let </span>actual_bytes = get_file_as_byte_vec(actual.to_input_file()).<span class="kw">await</span>;
+ <span class="kw">let </span>expected_bytes = get_file_as_byte_vec(expected).<span class="kw">await</span>;
+ <span class="macro">assert_eq!</span>(actual_bytes, expected_bytes);
+ }
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_uncompressed_empty_puffin_file_is_bit_identical_to_java_generated_file() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+
+ assert_files_are_bit_identical(
+ write_puffin_file(<span class="kw-2">&</span>temp_dir, Vec::new(), HashMap::new())
+ .<span class="kw">await
+ </span>.unwrap(),
+ java_empty_uncompressed_input_file(),
+ )
+ .<span class="kw">await
+ </span>}
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_uncompressed_metric_data_is_bit_identical_to_java_generated_file() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+ <span class="kw">let </span>blobs = <span class="macro">vec!</span>[blob_0(), blob_1()];
+ <span class="kw">let </span>blobs_with_compression = blobs_with_compression(blobs, CompressionCodec::None);
+
+ assert_files_are_bit_identical(
+ write_puffin_file(<span class="kw-2">&</span>temp_dir, blobs_with_compression, file_properties())
+ .<span class="kw">await
+ </span>.unwrap(),
+ java_uncompressed_metric_input_file(),
+ )
+ .<span class="kw">await
+ </span>}
+
+ <span class="attr">#[tokio::test]
+ </span><span class="kw">async fn </span>test_zstd_compressed_metric_data_is_bit_identical_to_java_generated_file() {
+ <span class="kw">let </span>temp_dir = TempDir::new().unwrap();
+ <span class="kw">let </span>blobs = <span class="macro">vec!</span>[blob_0(), blob_1()];
+ <span class="kw">let </span>blobs_with_compression = blobs_with_compression(blobs, CompressionCodec::Zstd);
+
+ assert_files_are_bit_identical(
+ write_puffin_file(<span class="kw-2">&</span>temp_dir, blobs_with_compression, file_properties())
+ .<span class="kw">await
+ </span>.unwrap(),
+ java_zstd_compressed_metric_input_file(),
+ )
+ .<span class="kw">await
+ </span>}
+}
+</code></pre></div></section></main></body></html>
\ No newline at end of file