You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by gi...@apache.org on 2024/05/08 09:50:31 UTC
(arrow-rs) branch asf-site updated: deploy: 4045fb5ba2dae222aab7365250d788ae8872e229
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/asf-site by this push:
new d36647ec647 deploy: 4045fb5ba2dae222aab7365250d788ae8872e229
d36647ec647 is described below
commit d36647ec64707a786370bf9224a351aa5ec422ba
Author: Jefffrey <Je...@users.noreply.github.com>
AuthorDate: Wed May 8 09:50:27 2024 +0000
deploy: 4045fb5ba2dae222aab7365250d788ae8872e229
---
arrow_csv/reader/fn.build_boolean_array.html | 2 +-
arrow_csv/reader/fn.build_decimal_array.html | 2 +-
arrow_csv/reader/fn.build_primitive_array.html | 2 +-
arrow_csv/reader/fn.build_timestamp_array.html | 2 +-
.../reader/fn.build_timestamp_array_impl.html | 2 +-
arrow_csv/reader/fn.infer_file_schema.html | 2 +-
arrow_csv/reader/fn.infer_reader_schema.html | 2 +-
arrow_csv/reader/fn.infer_schema_from_files.html | 2 +-
arrow_csv/reader/fn.parse.html | 2 +-
arrow_csv/reader/fn.parse_bool.html | 2 +-
arrow_csv/reader/index.html | 2 +-
arrow_csv/reader/records/index.html | 2 +-
arrow_csv/reader/records/struct.RecordDecoder.html | 16 +-
arrow_csv/reader/records/struct.StringRecord.html | 4 +-
arrow_csv/reader/records/struct.StringRecords.html | 4 +-
arrow_csv/reader/struct.BufReader.html | 10 +-
arrow_csv/reader/struct.Decoder.html | 10 +-
arrow_csv/reader/struct.Format.html | 16 +-
arrow_csv/reader/struct.ReaderBuilder.html | 31 +-
arrow_csv/reader/type.Bounds.html | 2 +-
arrow_csv/reader/type.Reader.html | 4 +-
search-index.js | 2 +-
search.desc/arrow_csv/arrow_csv-desc-0-.js | 2 +-
src/arrow_csv/reader/mod.rs.html | 312 ++++++++++++++++++++-
src/arrow_csv/reader/records.rs.html | 68 ++++-
type.impl/arrow_csv/reader/struct.BufReader.js | 2 +-
26 files changed, 441 insertions(+), 66 deletions(-)
diff --git a/arrow_csv/reader/fn.build_boolean_array.html b/arrow_csv/reader/fn.build_boolean_array.html
index 4df430a636a..a0fcc914bc4 100644
--- a/arrow_csv/reader/fn.build_boolean_array.html
+++ b/arrow_csv/reader/fn.build_boolean_array.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_boolean_array` fn in crate `arrow_csv`."><title>build_boolean_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSan [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_boolean_array` fn in crate `arrow_csv`."><title>build_boolean_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSan [...]
line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
rows: &<a class="struct" href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords">StringRecords</a><'_>,
col_idx: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
diff --git a/arrow_csv/reader/fn.build_decimal_array.html b/arrow_csv/reader/fn.build_decimal_array.html
index 6cdf390d341..ab3241a4fba 100644
--- a/arrow_csv/reader/fn.build_decimal_array.html
+++ b/arrow_csv/reader/fn.build_decimal_array.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_decimal_array` fn in crate `arrow_csv`."><title>build_decimal_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSan [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_decimal_array` fn in crate `arrow_csv`."><title>build_decimal_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSan [...]
_line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
rows: &<a class="struct" href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords">StringRecords</a><'_>,
col_idx: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
diff --git a/arrow_csv/reader/fn.build_primitive_array.html b/arrow_csv/reader/fn.build_primitive_array.html
index b3e83eb6ec7..178e9d63b8e 100644
--- a/arrow_csv/reader/fn.build_primitive_array.html
+++ b/arrow_csv/reader/fn.build_primitive_array.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_primitive_array` fn in crate `arrow_csv`."><title>build_primitive_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,Fir [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_primitive_array` fn in crate `arrow_csv`."><title>build_primitive_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,Fir [...]
line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
rows: &<a class="struct" href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords">StringRecords</a><'_>,
col_idx: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
diff --git a/arrow_csv/reader/fn.build_timestamp_array.html b/arrow_csv/reader/fn.build_timestamp_array.html
index b8217ea0b59..207074d4f63 100644
--- a/arrow_csv/reader/fn.build_timestamp_array.html
+++ b/arrow_csv/reader/fn.build_timestamp_array.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_timestamp_array` fn in crate `arrow_csv`."><title>build_timestamp_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,Fir [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_timestamp_array` fn in crate `arrow_csv`."><title>build_timestamp_array in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,Fir [...]
line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
rows: &<a class="struct" href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords">StringRecords</a><'_>,
col_idx: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
diff --git a/arrow_csv/reader/fn.build_timestamp_array_impl.html b/arrow_csv/reader/fn.build_timestamp_array_impl.html
index d9178c67842..a1fe8c250a4 100644
--- a/arrow_csv/reader/fn.build_timestamp_array_impl.html
+++ b/arrow_csv/reader/fn.build_timestamp_array_impl.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_timestamp_array_impl` fn in crate `arrow_csv`."><title>build_timestamp_array_impl in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `build_timestamp_array_impl` fn in crate `arrow_csv`."><title>build_timestamp_array_impl in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf [...]
line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
rows: &<a class="struct" href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords">StringRecords</a><'_>,
col_idx: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
diff --git a/arrow_csv/reader/fn.infer_file_schema.html b/arrow_csv/reader/fn.infer_file_schema.html
index 905a639d364..b51631718a9 100644
--- a/arrow_csv/reader/fn.infer_file_schema.html
+++ b/arrow_csv/reader/fn.infer_file_schema.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Infer the schema of a CSV file by reading through the first n records of the file, with `max_read_records` controlling the maximum number of records to read."><title>infer_file_schema in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdj [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Infer the schema of a CSV file by reading through the first n records of the file, with `max_read_records` controlling the maximum number of records to read."><title>infer_file_schema in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdj [...]
reader: R,
delimiter: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>,
max_read_records: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>>,
diff --git a/arrow_csv/reader/fn.infer_reader_schema.html b/arrow_csv/reader/fn.infer_reader_schema.html
index 94d6beeab49..64f2b3fc94f 100644
--- a/arrow_csv/reader/fn.infer_reader_schema.html
+++ b/arrow_csv/reader/fn.infer_reader_schema.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Infer schema of CSV records provided by struct that implements `Read` trait."><title>infer_reader_schema in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Infer schema of CSV records provided by struct that implements `Read` trait."><title>infer_reader_schema in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans [...]
reader: R,
delimiter: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>,
max_read_records: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>>,
diff --git a/arrow_csv/reader/fn.infer_schema_from_files.html b/arrow_csv/reader/fn.infer_schema_from_files.html
index 55cede38ec2..1fbc00e1a63 100644
--- a/arrow_csv/reader/fn.infer_schema_from_files.html
+++ b/arrow_csv/reader/fn.infer_schema_from_files.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Infer schema from a list of CSV files by reading through first n records with `max_read_records` controlling the maximum number of records to read."><title>infer_schema_from_files in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacen [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Infer schema from a list of CSV files by reading through first n records with `max_read_records` controlling the maximum number of records to read."><title>infer_schema_from_files in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacen [...]
files: &[<a class="struct" href="https://doc.rust-lang.org/nightly/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>],
delimiter: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>,
max_read_records: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>>,
diff --git a/arrow_csv/reader/fn.parse.html b/arrow_csv/reader/fn.parse.html
index 7bf8240d8c7..73134e63841 100644
--- a/arrow_csv/reader/fn.parse.html
+++ b/arrow_csv/reader/fn.parse.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Parses a slice of `StringRecords` into a RecordBatch"><title>parse in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSa [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Parses a slice of `StringRecords` into a RecordBatch"><title>parse in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSa [...]
rows: &<a class="struct" href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords">StringRecords</a><'_>,
fields: &Fields,
metadata: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="struct" href="https://doc.rust-lang.org/nightly/std/collections/hash/map/struct.HashMap.html" title="struct std::collections::hash::map::HashMap">HashMap</a><<a class="struct" href="https://doc.rust-lang.org/nightly/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>, <a class="struct" href="https://d [...]
diff --git a/arrow_csv/reader/fn.parse_bool.html b/arrow_csv/reader/fn.parse_bool.html
index b2dd2a634e6..c7cbe2333a2 100644
--- a/arrow_csv/reader/fn.parse_bool.html
+++ b/arrow_csv/reader/fn.parse_bool.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `parse_bool` fn in crate `arrow_csv`."><title>parse_bool in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141b [...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `parse_bool` fn in crate `arrow_csv`."><title>parse_bool in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141b [...]
\ No newline at end of file
diff --git a/arrow_csv/reader/index.html b/arrow_csv/reader/index.html
index bf4e346d20d..31e1610f552 100644
--- a/arrow_csv/reader/index.html
+++ b/arrow_csv/reader/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV Reader"><title>arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regu [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV Reader"><title>arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2,SourceCodePro-Regu [...]
<h2 id="basic-usage"><a class="doc-anchor" href="#basic-usage">§</a>Basic Usage</h2>
<p>This CSV reader allows CSV files to be read into the Arrow memory model. Records are
loaded in batches and are then converted from row-based data to columnar data.</p>
diff --git a/arrow_csv/reader/records/index.html b/arrow_csv/reader/records/index.html
index c5937f3ca3b..f7138c1c1d7 100644
--- a/arrow_csv/reader/records/index.html
+++ b/arrow_csv/reader/records/index.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `records` mod in crate `arrow_csv`."><title>arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ff [...]
\ No newline at end of file
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `records` mod in crate `arrow_csv`."><title>arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ff [...]
\ No newline at end of file
diff --git a/arrow_csv/reader/records/struct.RecordDecoder.html b/arrow_csv/reader/records/struct.RecordDecoder.html
index d0dc551bf0d..88c9092a65b 100644
--- a/arrow_csv/reader/records/struct.RecordDecoder.html
+++ b/arrow_csv/reader/records/struct.RecordDecoder.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="`RecordDecoder` provides a push-based interface to decoder `StringRecords`"><title>RecordDecoder in arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSan [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="`RecordDecoder` provides a push-based interface to decoder `StringRecords`"><title>RecordDecoder in arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSan [...]
delimiter: Reader,
num_columns: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
@@ -8,6 +8,7 @@
num_rows: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
data: <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>>,
data_len: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
+ truncated_rows: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>,
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p><a href="struct.RecordDecoder.html" title="struct arrow_csv::reader::records::RecordDecoder"><code>RecordDecoder</code></a> provides a push-based interface to decoder <a href="struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords"><code>StringRecords</code></a></p>
</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.delimiter" class="structfield section-header"><a href="#structfield.delimiter" class="anchor field">§</a><code>delimiter: Reader</code></span><span id="structfield.num_columns" class="structfield section-header"><a href="#structfield.num_columns" class="anchor field">§</a><code>num_columns: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/pr [...]
</div><span id="structfield.line_number" class="structfield section-header"><a href="#structfield.line_number" class="anchor field">§</a><code>line_number: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></code></span><div class="docblock"><p>The current line number</p>
@@ -19,16 +20,19 @@
</div><span id="structfield.data" class="structfield section-header"><a href="#structfield.data" class="anchor field">§</a><code>data: <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>></code></span><div class="docblock"><p>Decoded field data</p>
</div><span id="structfield.data_len" class="structfield section-header"><a href="#structfield.data_len" class="anchor field">§</a><code>data_len: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></code></span><div class="docblock"><p>Offsets into data</p>
<p>We track this independently of Vec to avoid re-zeroing memory</p>
-</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-RecordDecoder" class="impl"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#61-230">source</a><a href="#impl-RecordDecoder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.RecordDecoder.html" title="stru [...]
+</div><span id="structfield.truncated_rows" class="structfield section-header"><a href="#structfield.truncated_rows" class="anchor field">§</a><code>truncated_rows: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></code></span><div class="docblock"><p>Whether rows with less than expected columns are considered valid</p>
+<p>Default value is false
+When enabled fills in missing columns with null</p>
+</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-RecordDecoder" class="impl"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#67-246">source</a><a href="#impl-RecordDecoder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.RecordDecoder.html" title="stru [...]
&mut self,
input: &[<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>],
to_read: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>
) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><(<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>, <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>), ArrowError></h4></section></summary><div class="docblock"><p>Decodes records from <code>input</code> returning the number of records and bytes read</p>
<p>Note: this expects to be called with an empty <code>input</code> to signal EOF</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.len" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#158-160">source</a><h4 class="code-header">pub fn <a href="#method.len" class="fn">len</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></h4></section></summary><div class="docblock"><p>Returns the current number of buffered records</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.is_empty" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#163-165">source</a><h4 class="code-header">pub fn <a href="#method.is_empty" class="fn">is_empty</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></h4></section></summary><div class="docblock"><p>Returns true if the decoder is empty</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.clear" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#168-173">source</a><h4 class="code-header">pub fn <a href="#method.clear" class="fn">clear</a>(&mut self)</h4></section></summary><div class="docblock"><p>Clears the current contents of the decoder</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.flush" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#176-229">source</a><h4 class="code-header">pub fn <a href="#method.flush" class="fn">flush</a>(&mut self) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><<a class="struct" href="struct.StringRecords.html" titl [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.len" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#174-176">source</a><h4 class="code-header">pub fn <a href="#method.len" class="fn">len</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></h4></section></summary><div class="docblock"><p>Returns the current number of buffered records</p>
+</div></details><details class="toggle method-toggle" open><summary><section id="method.is_empty" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#179-181">source</a><h4 class="code-header">pub fn <a href="#method.is_empty" class="fn">is_empty</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></h4></section></summary><div class="docblock"><p>Returns true if the decoder is empty</p>
+</div></details><details class="toggle method-toggle" open><summary><section id="method.clear" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#184-189">source</a><h4 class="code-header">pub fn <a href="#method.clear" class="fn">clear</a>(&mut self)</h4></section></summary><div class="docblock"><p>Clears the current contents of the decoder</p>
+</div></details><details class="toggle method-toggle" open><summary><section id="method.flush" class="method"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#192-245">source</a><h4 class="code-header">pub fn <a href="#method.flush" class="fn">flush</a>(&mut self) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><<a class="struct" href="struct.StringRecords.html" titl [...]
</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-RecordDecoder" class="impl"><a class="src rightside" href="../../../src/arrow_csv/reader/records.rs.html#28">source</a><a href="#impl-Debug-for-RecordDecoder" class="anchor">§</a><h3 class="code-hea [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
diff --git a/arrow_csv/reader/records/struct.StringRecord.html b/arrow_csv/reader/records/struct.StringRecord.html
index 0feb034aa80..64b2e58ff3c 100644
--- a/arrow_csv/reader/records/struct.StringRecord.html
+++ b/arrow_csv/reader/records/struct.StringRecord.html
@@ -1,8 +1,8 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A single parsed, UTF-8 CSV record"><title>StringRecord in arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans- [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A single parsed, UTF-8 CSV record"><title>StringRecord in arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans- [...]
data: &'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>,
offsets: &'a [<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>],
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>A single parsed, UTF-8 CSV record</p>
-</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.data" class="structfield section-header"><a href="#structfield.data" class="anchor field">§</a><code>data: &'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a></code></span><span id="structfield.offsets" class="structfield section-header"><a href="#structfield.offsets" class="anchor field">§</a><code>offsets: [...]
+</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.data" class="structfield section-header"><a href="#structfield.data" class="anchor field">§</a><code>data: &'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a></code></span><span id="structfield.offsets" class="structfield section-header"><a href="#structfield.offsets" class="anchor field">§</a><code>offsets: [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow_mut" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217">source</a><a href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a href= [...]
diff --git a/arrow_csv/reader/records/struct.StringRecords.html b/arrow_csv/reader/records/struct.StringRecords.html
index 2cd188beb20..a16692226d4 100644
--- a/arrow_csv/reader/records/struct.StringRecords.html
+++ b/arrow_csv/reader/records/struct.StringRecords.html
@@ -1,10 +1,10 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A collection of parsed, UTF-8 CSV records"><title>StringRecords in arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2, [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A collection of parsed, UTF-8 CSV records"><title>StringRecords in arrow_csv::reader::records - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2, [...]
num_columns: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
num_rows: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
offsets: &'a [<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>],
data: &'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>,
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>A collection of parsed, UTF-8 CSV records</p>
-</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.num_columns" class="structfield section-header"><a href="#structfield.num_columns" class="anchor field">§</a><code>num_columns: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></code></span><span id="structfield.num_rows" class="structfield section-header"><a href="#structfield.num_rows" class="anchor field">§ [...]
+</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.num_columns" class="structfield section-header"><a href="#structfield.num_columns" class="anchor field">§</a><code>num_columns: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></code></span><span id="structfield.num_rows" class="structfield section-header"><a href="#structfield.num_rows" class="anchor field">§ [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow_mut" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217">source</a><a href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a href= [...]
diff --git a/arrow_csv/reader/struct.BufReader.html b/arrow_csv/reader/struct.BufReader.html
index 1ed4daa0249..b1973d2c20a 100644
--- a/arrow_csv/reader/struct.BufReader.html
+++ b/arrow_csv/reader/struct.BufReader.html
@@ -1,13 +1,13 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV file reader"><title>BufReader in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2, [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV file reader"><title>BufReader in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e4970d388.woff2, [...]
reader: R,
decoder: <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::Decoder">Decoder</a>,
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>CSV file reader</p>
</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.reader" class="structfield section-header"><a href="#structfield.reader" class="anchor field">§</a><code>reader: R</code></span><div class="docblock"><p>File reader</p>
</div><span id="structfield.decoder" class="structfield section-header"><a href="#structfield.decoder" class="anchor field">§</a><code>decoder: <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::Decoder">Decoder</a></code></span><div class="docblock"><p>The decoder</p>
-</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-BufReader%3CBufReader%3CR%3E%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#471-484">source</a><a href="#impl-BufReader%3CBufReader%3CR%3E%3E" class="anchor">§</a><h3 class="code-header">impl<R: <a class="trait" href="htt [...]
+</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-BufReader%3CBufReader%3CR%3E%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#484-497">source</a><a href="#impl-BufReader%3CBufReader%3CR%3E%3E" class="anchor">§</a><h3 class="code-header">impl<R: <a class="trait" href="htt [...]
record batches</p>
-</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-BufReader%3CR%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#486-504">source</a><a href="#impl-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl<R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead">BufRead</a>> <a class="struct" href="struct.BufReader.html [...]
- R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead">BufRead</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.fmt" class="method trait-impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#464-468">source</a><a href="#method.fmt" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly [...]
+</div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-BufReader%3CR%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#499-517">source</a><a href="#impl-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl<R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead">BufRead</a>> <a class="struct" href="struct.BufReader.html [...]
+ R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead">BufRead</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.fmt" class="method trait-impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#477-481">source</a><a href="#method.fmt" class="anchor">§</a><h4 class="code-header">fn <a href="https://doc.rust-lang.org/nightly [...]
&mut self
) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><[Self::<a class="associatedtype" href="https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html#associatedtype.Item" title="type core::iter::traits::iterator::Iterator::Item">Item</a>; <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.array.html">N</a>], <a class="struct" href="https://doc.rust-lang.o [...]
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h4></section></summary><span class="item-info"><div class="stab unstable"><span class="emoji">🔬</span><span>This is a nightly-only experimental API. (<code>iter_next_chunk</code>)</span></div></span><div class='docblock'>Advances the iterator and returns an array containing the next <code>N</code> values. <a href="https://doc.rust-lang.org/n [...]
@@ -184,7 +184,7 @@ greater than or equal to those of another. <a href="https://doc.rust-lang.org/ni
Self: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,
F: <a class="trait" href="https://doc.rust-lang.org/nightly/core/ops/function/trait.FnMut.html" title="trait core::ops::function::FnMut">FnMut</a>(Self::<a class="associatedtype" href="https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html#associatedtype.Item" title="type core::iter::traits::iterator::Iterator::Item">Item</a>) -> K,
K: <a class="trait" href="https://doc.rust-lang.org/nightly/core/cmp/trait.PartialOrd.html" title="trait core::cmp::PartialOrd">PartialOrd</a>,</div></h4></section></summary><span class="item-info"><div class="stab unstable"><span class="emoji">🔬</span><span>This is a nightly-only experimental API. (<code>is_sorted</code>)</span></div></span><div class='docblock'>Checks if the elements of this iterator are sorted using the given key extraction
-function. <a href="https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html#method.is_sorted_by_key">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-RecordBatchReader-for-BufReader%3CR%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#514-518">source</a><a href="#impl-RecordBatchReader-for-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl&l [...]
+function. <a href="https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html#method.is_sorted_by_key">Read more</a></div></details></div></details><details class="toggle implementors-toggle" open><summary><section id="impl-RecordBatchReader-for-BufReader%3CR%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#527-531">source</a><a href="#impl-RecordBatchReader-for-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl&l [...]
R: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Freeze.html" title="trait core::marker::Freeze">Freeze</a>,</div></h3></section><section id="impl-RefUnwindSafe-for-BufReader%3CR%3E" class="impl"><a href="#impl-RefUnwindSafe-for-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl<R> <a class="trait" href="https://doc.rust-lang.org/nightly/core/panic/unwind_safe/trait.RefUnwindSafe.html" title="trait core::panic::unwind_safe::RefUnwind [...]
R: <a class="trait" href="https://doc.rust-lang.org/nightly/core/panic/unwind_safe/trait.RefUnwindSafe.html" title="trait core::panic::unwind_safe::RefUnwindSafe">RefUnwindSafe</a>,</div></h3></section><section id="impl-Send-for-BufReader%3CR%3E" class="impl"><a href="#impl-Send-for-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl<R> <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Send.html" title="trait core::marker::Send">Send</a> [...]
R: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Send.html" title="trait core::marker::Send">Send</a>,</div></h3></section><section id="impl-Sync-for-BufReader%3CR%3E" class="impl"><a href="#impl-Sync-for-BufReader%3CR%3E" class="anchor">§</a><h3 class="code-header">impl<R> <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sync.html" title="trait core::marker::Sync">Sync</a> for <a class="struct" href="struct.BufReader.html" [...]
diff --git a/arrow_csv/reader/struct.Decoder.html b/arrow_csv/reader/struct.Decoder.html
index 5128a763879..41c18f4f516 100644
--- a/arrow_csv/reader/struct.Decoder.html
+++ b/arrow_csv/reader/struct.Decoder.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A push-based interface for decoding CSV data from an arbitrary byte stream"><title>Decoder in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c1 [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="A push-based interface for decoding CSV data from an arbitrary byte stream"><title>Decoder in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c1 [...]
schema: SchemaRef,
projection: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>>>,
batch_size: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
@@ -45,19 +45,19 @@ object storage</p>
</div><span id="structfield.end" class="structfield section-header"><a href="#structfield.end" class="anchor field">§</a><code>end: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></code></span><div class="docblock"><p>End line number</p>
</div><span id="structfield.record_decoder" class="structfield section-header"><a href="#structfield.record_decoder" class="anchor field">§</a><code>record_decoder: <a class="struct" href="records/struct.RecordDecoder.html" title="struct arrow_csv::reader::records::RecordDecoder">RecordDecoder</a></code></span><div class="docblock"><p>A decoder for <a href="records/struct.StringRecords.html" title="struct arrow_csv::reader::records::StringRecords"><code>StringRecords</code></a></p>
</div><span id="structfield.null_regex" class="structfield section-header"><a href="#structfield.null_regex" class="anchor field">§</a><code>null_regex: <a class="struct" href="struct.NullRegex.html" title="struct arrow_csv::reader::NullRegex">NullRegex</a></code></span><div class="docblock"><p>Check if the string matches this pattern for <code>NULL</code>.</p>
-</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Decoder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#586-639">source</a><a href="#impl-Decoder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::De [...]
+</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Decoder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#599-652">source</a><a href="#impl-Decoder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::De [...]
<p>This method returns once <code>batch_size</code> objects have been parsed since the
last call to <a href="struct.Decoder.html#method.flush" title="method arrow_csv::reader::Decoder::flush"><code>Self::flush</code></a>, or <code>buf</code> is exhausted. Any remaining bytes
should be included in the next call to <a href="struct.Decoder.html#method.decode" title="method arrow_csv::reader::Decoder::decode"><code>Self::decode</code></a></p>
<p>There is no requirement that <code>buf</code> contains a whole number of records, facilitating
integration with arbitrary byte streams, such as that yielded by <a href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead"><code>BufRead</code></a> or
network sources such as object storage</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.flush" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#617-633">source</a><h4 class="code-header">pub fn <a href="#method.flush" class="fn">flush</a>(&mut self) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><<a class="enum" href="https://doc.rust-lang.org/nightly/core/o [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.flush" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#630-646">source</a><h4 class="code-header">pub fn <a href="#method.flush" class="fn">flush</a>(&mut self) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><<a class="enum" href="https://doc.rust-lang.org/nightly/core/o [...]
<p>This should only be called after <a href="struct.Decoder.html#method.decode" title="method arrow_csv::reader::Decoder::decode"><code>Self::decode</code></a> has returned <code>Ok(0)</code>,
otherwise may return an error if part way through decoding a record</p>
<p>Returns <code>Ok(None)</code> if no buffered data</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.capacity" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#636-638">source</a><h4 class="code-header">pub fn <a href="#method.capacity" class="fn">capacity</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></h4></section></summary><div class="docblock"><p>Returns the number of records that can be rea [...]
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-Decoder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#559">source</a><a href="#impl-Debug-for-Decoder" class="anchor">§</a><h3 class="code-header">impl <a class [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.capacity" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#649-651">source</a><h4 class="code-header">pub fn <a href="#method.capacity" class="fn">capacity</a>(&self) -> <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a></h4></section></summary><div class="docblock"><p>Returns the number of records that can be rea [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-Decoder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#572">source</a><a href="#impl-Debug-for-Decoder" class="anchor">§</a><h3 class="code-header">impl <a class [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow_mut" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217">source</a><a href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a href= [...]
diff --git a/arrow_csv/reader/struct.Format.html b/arrow_csv/reader/struct.Format.html
index d30197586ca..736209cbbc4 100644
--- a/arrow_csv/reader/struct.Format.html
+++ b/arrow_csv/reader/struct.Format.html
@@ -1,13 +1,19 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="The format specification for the CSV file"><title>Format in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium- [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="The format specification for the CSV file"><title>Format in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium- [...]
header: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>,
delimiter: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>>,
escape: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>>,
quote: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>>,
terminator: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>>,
null_regex: <a class="struct" href="struct.NullRegex.html" title="struct arrow_csv::reader::NullRegex">NullRegex</a>,
+ truncated_rows: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>,
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p>The format specification for the CSV file</p>
-</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.header" class="structfield section-header"><a href="#structfield.header" class="anchor field">§</a><code>header: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></code></span><span id="structfield.delimiter" class="structfield section-header"><a href="#structfield.delimiter" class="anchor field">§</a><code>delim [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.infer_schema" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#274-326">source</a><h4 class="code-header">pub fn <a href="#method.infer_schema" class="fn">infer_schema</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.Read.html" title="trait std::io::Read">Read</a>>(
+</div></details><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.header" class="structfield section-header"><a href="#structfield.header" class="anchor field">§</a><code>header: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a></code></span><span id="structfield.delimiter" class="structfield section-header"><a href="#structfield.delimiter" class="anchor field">§</a><code>delim [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_truncated_rows" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#275-278">source</a><h4 class="code-header">pub fn <a href="#method.with_truncated_rows" class="fn">with_truncated_rows</a>(self, allow: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>) -> Self</h4></section></summary><div class="docblock"><p>Whethe [...]
+<p>By default this is set to <code>false</code> and will error if the CSV rows have different lengths.
+When set to true then it will allow records with less than the expected number of columns
+and fill the missing columns with nulls. If the record’s schema is not nullable, then it
+will still return an error.</p>
+</div></details><details class="toggle method-toggle" open><summary><section id="method.infer_schema" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#286-338">source</a><h4 class="code-header">pub fn <a href="#method.infer_schema" class="fn">infer_schema</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.Read.html" title="trait std::io::Read">Read</a>>(
&self,
reader: R,
max_records: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>>
@@ -15,8 +21,8 @@
<p>If <code>max_records</code> is <code>None</code>, all records will be read, otherwise up to <code>max_records</code>
records are read to infer the schema</p>
<p>Returns inferred schema and number of records read</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.build_reader" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#329-344">source</a><h4 class="code-header">fn <a href="#method.build_reader" class="fn">build_reader</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.Read.html" title="trait std::io::Read">Read</a>>(&self, reader: R) -> Reader<R></h4></section></summary><div [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.build_parser" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#347-361">source</a><h4 class="code-header">fn <a href="#method.build_parser" class="fn">build_parser</a>(&self) -> Reader</h4></section></summary><div class="docblock"><p>Build a [<code>csv_core::Reader</code>] for this <a href="struct.Format.html" title="struct arrow_csv::reader::Format"><code>Fo [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.build_reader" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#341-357">source</a><h4 class="code-header">fn <a href="#method.build_reader" class="fn">build_reader</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.Read.html" title="trait std::io::Read">Read</a>>(&self, reader: R) -> Reader<R></h4></section></summary><div [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.build_parser" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#360-374">source</a><h4 class="code-header">fn <a href="#method.build_parser" class="fn">build_parser</a>(&self) -> Reader</h4></section></summary><div class="docblock"><p>Build a [<code>csv_core::Reader</code>] for this <a href="struct.Format.html" title="struct arrow_csv::reader::Format"><code>Fo [...]
</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Clone-for-Format" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#226">source</a><a href="#impl-Clone-for-Format" class="anchor">§</a><h3 class="code-header">impl <a class=" [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
diff --git a/arrow_csv/reader/struct.ReaderBuilder.html b/arrow_csv/reader/struct.ReaderBuilder.html
index 9b3557d761d..35db6a6a90c 100644
--- a/arrow_csv/reader/struct.ReaderBuilder.html
+++ b/arrow_csv/reader/struct.ReaderBuilder.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV file reader builder"><title>ReaderBuilder in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV file reader builder"><title>ReaderBuilder in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium-8f9a781e497 [...]
schema: SchemaRef,
format: <a class="struct" href="struct.Format.html" title="struct arrow_csv::reader::Format">Format</a>,
batch_size: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>,
@@ -11,7 +11,7 @@
<p>The default batch size when using the <code>ReaderBuilder</code> is 1024 records</p>
</div><span id="structfield.bounds" class="structfield section-header"><a href="#structfield.bounds" class="anchor field">§</a><code>bounds: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><(<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>, <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>)></code></span><di [...]
</div><span id="structfield.projection" class="structfield section-header"><a href="#structfield.projection" class="anchor field">§</a><code>projection: <a class="enum" href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html" title="enum core::option::Option">Option</a><<a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usi [...]
-</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ReaderBuilder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1026-1163">source</a><a href="#impl-ReaderBuilder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.ReaderBuilder.html" title="struct a [...]
+</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-ReaderBuilder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1039-1191">source</a><a href="#impl-ReaderBuilder" class="anchor">§</a><h3 class="code-header">impl <a class="struct" href="struct.ReaderBuilder.html" title="struct a [...]
<p>To convert a builder into a reader, call <code>ReaderBuilder::build</code></p>
<h5 id="example"><a class="doc-anchor" href="#example">§</a>Example</h5>
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="kw">let </span><span class="kw-2">mut </span>file = File::open(<span class="string">"test/data/uk_cities_with_headers.csv"</span>).unwrap();
@@ -21,23 +21,28 @@ file.rewind().unwrap();
<span class="comment">// create a builder
</span>ReaderBuilder::new(Arc::new(schema)).build(file).unwrap();</code></pre></div>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_header" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1067-1070">source</a><h4 class="code-header">pub fn <a href="#method.with_header" class="fn">with_header</a>(self, has_header: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>) -> Self</h4></section></summary><div class="docblock"><p>Set whether the CSV fil [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_format" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1073-1076">source</a><h4 class="code-header">pub fn <a href="#method.with_format" class="fn">with_format</a>(self, format: <a class="struct" href="struct.Format.html" title="struct arrow_csv::reader::Format">Format</a>) -> Self</h4></section></summary><div class="docblock"><p>Overrides the <a href="str [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_delimiter" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1079-1082">source</a><h4 class="code-header">pub fn <a href="#method.with_delimiter" class="fn">with_delimiter</a>(self, delimiter: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>) -> Self</h4></section></summary><div class="docblock"><p>Set the CSV file’s [...]
-</div></details><section id="method.with_escape" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1084-1087">source</a><h4 class="code-header">pub fn <a href="#method.with_escape" class="fn">with_escape</a>(self, escape: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>) -> Self</h4></section><section id="method.with_quote" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1 [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_batch_size" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1106-1109">source</a><h4 class="code-header">pub fn <a href="#method.with_batch_size" class="fn">with_batch_size</a>(self, batch_size: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>) -> Self</h4></section></summary><div class="docblock"><p>Set the b [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_bounds" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1113-1116">source</a><h4 class="code-header">pub fn <a href="#method.with_bounds" class="fn">with_bounds</a>(self, start: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>, end: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize. [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_header" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1080-1083">source</a><h4 class="code-header">pub fn <a href="#method.with_header" class="fn">with_header</a>(self, has_header: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>) -> Self</h4></section></summary><div class="docblock"><p>Set whether the CSV fil [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_format" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1086-1089">source</a><h4 class="code-header">pub fn <a href="#method.with_format" class="fn">with_format</a>(self, format: <a class="struct" href="struct.Format.html" title="struct arrow_csv::reader::Format">Format</a>) -> Self</h4></section></summary><div class="docblock"><p>Overrides the <a href="str [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_delimiter" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1092-1095">source</a><h4 class="code-header">pub fn <a href="#method.with_delimiter" class="fn">with_delimiter</a>(self, delimiter: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>) -> Self</h4></section></summary><div class="docblock"><p>Set the CSV file’s [...]
+</div></details><section id="method.with_escape" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1097-1100">source</a><h4 class="code-header">pub fn <a href="#method.with_escape" class="fn">with_escape</a>(self, escape: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.u8.html">u8</a>) -> Self</h4></section><section id="method.with_quote" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1 [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_batch_size" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1119-1122">source</a><h4 class="code-header">pub fn <a href="#method.with_batch_size" class="fn">with_batch_size</a>(self, batch_size: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>) -> Self</h4></section></summary><div class="docblock"><p>Set the b [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_bounds" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1126-1129">source</a><h4 class="code-header">pub fn <a href="#method.with_bounds" class="fn">with_bounds</a>(self, start: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>, end: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize. [...]
<code>start</code> and <code>end</code> are line numbers.</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.with_projection" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1119-1122">source</a><h4 class="code-header">pub fn <a href="#method.with_projection" class="fn">with_projection</a>(self, projection: <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.r [...]
-</div></details><details class="toggle method-toggle" open><summary><section id="method.build" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1128-1130">source</a><h4 class="code-header">pub fn <a href="#method.build" class="fn">build</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.Read.html" title="trait std::io::Read">Read</a>>(self, reader: R) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/res [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_projection" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1132-1135">source</a><h4 class="code-header">pub fn <a href="#method.with_projection" class="fn">with_projection</a>(self, projection: <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><<a class="primitive" href="https://doc.r [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.with_truncated_rows" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1143-1146">source</a><h4 class="code-header">pub fn <a href="#method.with_truncated_rows" class="fn">with_truncated_rows</a>(self, allow: <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.bool.html">bool</a>) -> Self</h4></section></summary><div class="docblock"><p>Whet [...]
+<p>By default this is set to <code>false</code> and will error if the CSV rows have different lengths.
+When set to true then it will allow records with less than the expected number of columns
+and fill the missing columns with nulls. If the record’s schema is not nullable, then it
+will still return an error.</p>
+</div></details><details class="toggle method-toggle" open><summary><section id="method.build" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1152-1154">source</a><h4 class="code-header">pub fn <a href="#method.build" class="fn">build</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.Read.html" title="trait std::io::Read">Read</a>>(self, reader: R) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/res [...]
<p>If <code>R: BufRead</code> consider using <a href="struct.ReaderBuilder.html#method.build_buffered" title="method arrow_csv::reader::ReaderBuilder::build_buffered"><code>Self::build_buffered</code></a> to avoid unnecessary additional
buffering, as internally this method wraps <code>reader</code> in <a href="https://doc.rust-lang.org/nightly/std/io/buffered/bufreader/struct.BufReader.html" title="struct std::io::buffered::bufreader::BufReader"><code>std::io::BufReader</code></a></p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.build_buffered" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1133-1138">source</a><h4 class="code-header">pub fn <a href="#method.build_buffered" class="fn">build_buffered</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead">BufRead</a>>(
+</div></details><details class="toggle method-toggle" open><summary><section id="method.build_buffered" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1157-1162">source</a><h4 class="code-header">pub fn <a href="#method.build_buffered" class="fn">build_buffered</a><R: <a class="trait" href="https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html" title="trait std::io::BufRead">BufRead</a>>(
self,
reader: R
) -> <a class="enum" href="https://doc.rust-lang.org/nightly/core/result/enum.Result.html" title="enum core::result::Result">Result</a><<a class="struct" href="struct.BufReader.html" title="struct arrow_csv::reader::BufReader">BufReader</a><R>, ArrowError></h4></section></summary><div class="docblock"><p>Create a new <code>BufReader</code> from a buffered reader</p>
-</div></details><details class="toggle method-toggle" open><summary><section id="method.build_decoder" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1141-1162">source</a><h4 class="code-header">pub fn <a href="#method.build_decoder" class="fn">build_decoder</a>(self) -> <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::Decoder">Decoder</a></h4></section></summary><div class="docblock"><p>Builds a decoder that can be us [...]
-</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-ReaderBuilder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1010">source</a><a href="#impl-Debug-for-ReaderBuilder" class="anchor">§</a><h3 class="code-header"> [...]
+</div></details><details class="toggle method-toggle" open><summary><section id="method.build_decoder" class="method"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1165-1190">source</a><h4 class="code-header">pub fn <a href="#method.build_decoder" class="fn">build_decoder</a>(self) -> <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::Decoder">Decoder</a></h4></section></summary><div class="docblock"><p>Builds a decoder that can be us [...]
+</div></details></div></details></div><h2 id="trait-implementations" class="section-header">Trait Implementations<a href="#trait-implementations" class="anchor">§</a></h2><div id="trait-implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-Debug-for-ReaderBuilder" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#1023">source</a><a href="#impl-Debug-for-ReaderBuilder" class="anchor">§</a><h3 class="code-header"> [...]
T: 'static + ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.type_id" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/any.rs.html#141">source</a><a href="#method.type_id" class="anchor">§</a><h4 class="code-header">fn <a href [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#210">source</a><a href="#method.borrow" class="anchor">§</a><h4 class="code-header">fn <a href="https:/ [...]
T: ?<a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a>,</div></h3></section></summary><div class="impl-items"><details class="toggle method-toggle" open><summary><section id="method.borrow_mut" class="method trait-impl"><a class="src rightside" href="https://doc.rust-lang.org/nightly/src/core/borrow.rs.html#217">source</a><a href="#method.borrow_mut" class="anchor">§</a><h4 class="code-header">fn <a href= [...]
diff --git a/arrow_csv/reader/type.Bounds.html b/arrow_csv/reader/type.Bounds.html
index 9833bf56b20..c757e6ba1b4 100644
--- a/arrow_csv/reader/type.Bounds.html
+++ b/arrow_csv/reader/type.Bounds.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `Bounds` type in crate `arrow_csv`."><title>Bounds in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843f [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="API documentation for the Rust `Bounds` type in crate `arrow_csv`."><title>Bounds in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843f [...]
None,
Some((<a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>, <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.usize.html">usize</a>)),
}</code></pre><h2 id="variants" class="variants section-header">Variants<a href="#variants" class="anchor">§</a></h2><div class="variants"><section id="variant.None" class="variant"><a href="#variant.None" class="anchor">§</a><span class="since rightside" title="Stable since Rust version 1.0.0">1.0.0</span><h3 class="code-header">None</h3></section><div class="docblock"><p>No value.</p>
diff --git a/arrow_csv/reader/type.Reader.html b/arrow_csv/reader/type.Reader.html
index a64767b3c74..7946984c2ed 100644
--- a/arrow_csv/reader/type.Reader.html
+++ b/arrow_csv/reader/type.Reader.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV file reader using `std::io::BufReader`"><title>Reader in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="CSV file reader using `std::io::BufReader`"><title>Reader in arrow_csv::reader - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-46f98efaafac5295.ttf.woff2,FiraSans-Regular-018c141bf0843ffd.woff2,FiraSans-Medium [...]
</div></details><h2 id="aliased-type" class="section-header">Aliased Type<a href="#aliased-type" class="anchor">§</a></h2><pre class="rust item-decl"><code>struct Reader<R> {
reader: <a class="struct" href="https://doc.rust-lang.org/nightly/std/io/buffered/bufreader/struct.BufReader.html" title="struct std::io::buffered::bufreader::BufReader">BufReader</a><R>,
decoder: <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::Decoder">Decoder</a>,
}</code></pre><h2 id="fields" class="fields section-header">Fields<a href="#fields" class="anchor">§</a></h2><span id="structfield.reader" class="structfield section-header"><a href="#structfield.reader" class="anchor field">§</a><code>reader: <a class="struct" href="https://doc.rust-lang.org/nightly/std/io/buffered/bufreader/struct.BufReader.html" title="struct std::io::buffered::bufreader::BufReader">BufReader</a><R></code></span><div class="docblock"><p>File reader</p>
</div><span id="structfield.decoder" class="structfield section-header"><a href="#structfield.decoder" class="anchor field">§</a><code>decoder: <a class="struct" href="struct.Decoder.html" title="struct arrow_csv::reader::Decoder">Decoder</a></code></span><div class="docblock"><p>The decoder</p>
-</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-BufReader%3CBufReader%3CR%3E%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#471-484">source</a><a href="#impl-BufReader%3CBufReader%3CR%3E%3E" class="anchor">§</a><h3 class="code-header">impl<R: <a class="trait" href="htt [...]
+</div><h2 id="implementations" class="section-header">Implementations<a href="#implementations" class="anchor">§</a></h2><div id="implementations-list"><details class="toggle implementors-toggle" open><summary><section id="impl-BufReader%3CBufReader%3CR%3E%3E" class="impl"><a class="src rightside" href="../../src/arrow_csv/reader/mod.rs.html#484-497">source</a><a href="#impl-BufReader%3CBufReader%3CR%3E%3E" class="anchor">§</a><h3 class="code-header">impl<R: <a class="trait" href="htt [...]
record batches</p>
</div></details></div></details></div><script src="../../type.impl/arrow_csv/reader/struct.BufReader.js" data-self-path="arrow_csv::reader::Reader" async></script></section></div></main></body></html>
\ No newline at end of file
diff --git a/search-index.js b/search-index.js
index fa118a306c4..732b9bbeeef 100644
--- a/search-index.js
+++ b/search-index.js
@@ -5,7 +5,7 @@ var searchIndex = new Map(JSON.parse('[\
["arrow_avro",{"t":"CCCCFFPPGPPPPPPPPPPPGFPPPPPPNNNNNNNNNNNNNNNNNNNONONNNNNNNNNNNNNNNNNNNHOOOONNNNNNNNNNNNNNNNNNNNNNSGPPPNNNNNNNNNNNNNCCHHCFFGPPPPPNNNNNNOOONNNNNNNNNNONNNOONNNNNNNNNOPPPFFGPPSPPPPNNNNNNONNNNNNNNNNNNNNNNOOOONONOONNNNNNNONNNOFNNNNNONNONNNFPFPPPGPFPFFPPPPFPPPGFPPSGPFPGPPOOOOOOOOOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNOONNNNNNNNNNNNOOONNNNNNNNNNNNNONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNOOOOOOOOONNNNNNNNNNNNOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNOONNNNNNNNNNNNO","n":[" [...]
["arrow_buffer",{"t":"KKCMCCCCCCCCMOFOCMMMMMCSKPGPCNNNNNNNNSGPPTTTTFTNNNNNNNNNNNNNNNNNNNNNNNNNNNNQQCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNOFNNNNNONNNNHNHNNNNNNNNHNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNFHHHNNNNHHHHNHHHNHHNNNFFFIOOOOOOOOOOOOOOOFFFOOOOOOOOOOHHHHHHHHHHHFFFFFFFHHHCOOOHHHHHOOCOOOOOCCOCOOCOOCOCFNNNNNNONNNNNNNNNNNNNNNNNNNONNNNNONNNNNNNNNNNNFNNNNNNNNNNNNNONNNNNNNNNNNNNNNNNNNNNNOONNNNNNNNNFFNNNNNNNNNNNHONNNNNNNNNNNNNNNNNNNNNNNNNNONOOONNNNNNNNNNNNNNNNNNNNNNNFNNNONNNNNNNNNNNNNNNNNNNNNNNNNONNNNNN [...]
["arrow_cast",{"t":"CCCCCSSSSRRKHHMNNNNNNNFHHHNNHHHHHHHHHHHHHHHHHHNNCNCNNNNONNNCHHHOCHHNNNHNKHHHHHHHHHHMHHMMHHHHHHHHHHHHHHHHHHHHHFFPKKGPIPGFIPPPPPRIFOHHNNNNNNNNNNNNNNNNHOOQNQQONNNNNNNNNOONNNNNNNNNNONNNNNNHHNOOMQQOOOQOQOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNMMNNHHPPPSSPSFFFGPPPPISSSSPKPFPPNNNNNNNNNNNNNNNNONOONNNNONNNNNNNONNNNNHOOONNNMNHHHNHHHHHQHHHHNNNNNNHNNNNNNNNNNNNNNNNHHHHHHHH","n":["base64","cast","display","parse","pretty","BASE64_STANDARD","BASE64_STANDARD_NO_PAD","BASE64_URL_SAFE","BASE [...]
-["arrow_csv",{"t":"EEEEEHCCIFFFFPFFIFPOOONNNNNNNNNNNNNNONHNHNNHNHHNNNNNNNNOONNNONOONNNNNNONNNNNNNNOHHNHNNNNNNNNNONNOOOHHOOONOOOCNNNOOONNNONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNSSFFFNNNNNNNNNOOOOONONNNNNNNNNNNNNNNNONOOOOOOOONNNNNNNNNNSFFONNNNNNNNNOONOONNONONONNNNOONNNNNNNOONONOONOOOONNNNNNNNNNNNNNNNNNNNO","n":["Reader","ReaderBuilder","Writer","WriterBuilder","infer_schema_from_files","map_csv_error","reader","writer","Bounds","BufReader","Decoder","Format","InferredDataType","None","Null [...]
+["arrow_csv",{"t":"EEEEEHCCIFFFFPFFIFPOOONNNNNNNNNNNNNNONHNHNNHNHHNNNNNNNNOONNNONOONNNNNNONNNNNNNNOHHNHNNNNNNNNNONNOOOHHOOONOOOCNNNOOONNNOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNSSFFFNNNNNNNNNOOOOONONNNNNNNNNNNNNNNNONOOOOOOOONONNNNNNNNNSFFONNNNNNNNNOONOONNONONONNNNOONNNNNNNOONONOONOOOONNNNNNNNNNNNNNNNNNNNO","n":["Reader","ReaderBuilder","Writer","WriterBuilder","infer_schema_from_files","map_csv_error","reader","writer","Bounds","BufReader","Decoder","Format","InferredDataType","None"," [...]
["arrow_data",{"t":"PFFIPGFFPPOCCOOOOCOOOCOOCCCHOOOOOOOOOOOOOCHHOOOFFFNNNNNNOOOOOOONNNNNNNNNNNOOONNNNNNNHNNNNNNNNNHFNNNONNNNNNNOOONNNNHHHPFFIPGFPPNNNNONNNNNNNNNNNOOONNNNNONNNNOONNHHNNOONNNNNNNNNNNNNNNNNNNNNNNNNHNNOONNHNNNNNNNNNONNONNOONNOOOONNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNOOOSSSSSSSSSHHCCCHHHCCCCCCCCCCHHHHHHHHSHHHHHHHHHHHHHFFHNNNNNOOONOONOONNNNNNNNNNNOOONNONNNOOOHNNNNNNPPGPIIIPFPFOCNNNNNNOOHHHHHONNOOONONOOCCNNNNNNNNNNNNNOCNCNONOHCCNNNNNNNNNNCCCONHHHHHHHHHHHHHHHHHHHHHHHHH","n":["AlwaysNull [...]
["arrow_file_to_stream",{"t":"FNNNNNNONNNNNNHNNNNNN","n":["Args","augment_args","augment_args_for_update","borrow","borrow_mut","command","command_for_update","file_name","fmt","from","from_arg_matches","from_arg_matches_mut","group_id","into","main","try_from","try_into","type_id","update_from_arg_matches","update_from_arg_matches_mut","vzip"],"q":[[0,"arrow_file_to_stream"],[21,"clap_builder::builder::command"],[22,"core::fmt"],[23,"clap_builder::parser::matches::arg_matches"],[24,"cla [...]
["arrow_flight",{"t":"FFIFFFGPPFFPEFFFFFFFFPPFFFFFFFPOOOOOONNNNCOOOCNNOCOOCOOOCOOOHHCCNNNCOONNNNHONOOOOOOOOOOOHCOOOOOCNNNNNOOONNOOCNNFFFNNNNNNNNNNNNONNNNNNNNONNNNNNNNNNNNNNONNNNNNNNNNONOONNNNNNNNNNNNNNFGFFFPPPNNNNNNNNNNNOONNNNNNNNNNNNNOOONNNNNNNNNNNNNNNNNNNONNONNOONONNNNNNNNNNNNNNNNNNNNNNNNGFFFSPPOONNNNNNNNNONOOOOOONNNNONNNNNNNHHONNNNNNNNNOONNNOONHHONNOOHNNNNNNNNNNNNNNNNNNNNNNNPPPPGPPPIPNNNNNNNNNNNNNNNNNPGPPFORRRRKFRRROMMMMMMMOMMOOMOFFFFFGPPFFFFFFFFFPFFFFFFPOOOONOONNNNNNNNNNNNNNNNNNNNNNN [...]
diff --git a/search.desc/arrow_csv/arrow_csv-desc-0-.js b/search.desc/arrow_csv/arrow_csv-desc-0-.js
index c24ddf1a149..5ba31778bfc 100644
--- a/search.desc/arrow_csv/arrow_csv-desc-0-.js
+++ b/search.desc/arrow_csv/arrow_csv-desc-0-.js
@@ -1 +1 @@
-searchState.loadedDescShard("arrow_csv", 0, "Transfer data between the Arrow memory format and CSV …\nCSV Reader\nCSV Writer\nCSV file reader\nA push-based interface for decoding CSV data from an …\nThe format specification for the CSV file\nNo value.\nA wrapper over <code>Option<Regex></code> to check if the value is <code>NULL</code>.\nOrder should match <code>InferredDataType</code>\nCSV file reader using <code>std::io::BufReader</code>\nCSV file reader builder\nSome value of ty [...]
\ No newline at end of file
+searchState.loadedDescShard("arrow_csv", 0, "Transfer data between the Arrow memory format and CSV …\nCSV Reader\nCSV Writer\nCSV file reader\nA push-based interface for decoding CSV data from an …\nThe format specification for the CSV file\nNo value.\nA wrapper over <code>Option<Regex></code> to check if the value is <code>NULL</code>.\nOrder should match <code>InferredDataType</code>\nCSV file reader using <code>std::io::BufReader</code>\nCSV file reader builder\nSome value of ty [...]
\ No newline at end of file
diff --git a/src/arrow_csv/reader/mod.rs.html b/src/arrow_csv/reader/mod.rs.html
index ce7f97386e5..0f810a10e5e 100644
--- a/src/arrow_csv/reader/mod.rs.html
+++ b/src/arrow_csv/reader/mod.rs.html
@@ -2382,6 +2382,161 @@
<a href="#2382" id="2382">2382</a>
<a href="#2383" id="2383">2383</a>
<a href="#2384" id="2384">2384</a>
+<a href="#2385" id="2385">2385</a>
+<a href="#2386" id="2386">2386</a>
+<a href="#2387" id="2387">2387</a>
+<a href="#2388" id="2388">2388</a>
+<a href="#2389" id="2389">2389</a>
+<a href="#2390" id="2390">2390</a>
+<a href="#2391" id="2391">2391</a>
+<a href="#2392" id="2392">2392</a>
+<a href="#2393" id="2393">2393</a>
+<a href="#2394" id="2394">2394</a>
+<a href="#2395" id="2395">2395</a>
+<a href="#2396" id="2396">2396</a>
+<a href="#2397" id="2397">2397</a>
+<a href="#2398" id="2398">2398</a>
+<a href="#2399" id="2399">2399</a>
+<a href="#2400" id="2400">2400</a>
+<a href="#2401" id="2401">2401</a>
+<a href="#2402" id="2402">2402</a>
+<a href="#2403" id="2403">2403</a>
+<a href="#2404" id="2404">2404</a>
+<a href="#2405" id="2405">2405</a>
+<a href="#2406" id="2406">2406</a>
+<a href="#2407" id="2407">2407</a>
+<a href="#2408" id="2408">2408</a>
+<a href="#2409" id="2409">2409</a>
+<a href="#2410" id="2410">2410</a>
+<a href="#2411" id="2411">2411</a>
+<a href="#2412" id="2412">2412</a>
+<a href="#2413" id="2413">2413</a>
+<a href="#2414" id="2414">2414</a>
+<a href="#2415" id="2415">2415</a>
+<a href="#2416" id="2416">2416</a>
+<a href="#2417" id="2417">2417</a>
+<a href="#2418" id="2418">2418</a>
+<a href="#2419" id="2419">2419</a>
+<a href="#2420" id="2420">2420</a>
+<a href="#2421" id="2421">2421</a>
+<a href="#2422" id="2422">2422</a>
+<a href="#2423" id="2423">2423</a>
+<a href="#2424" id="2424">2424</a>
+<a href="#2425" id="2425">2425</a>
+<a href="#2426" id="2426">2426</a>
+<a href="#2427" id="2427">2427</a>
+<a href="#2428" id="2428">2428</a>
+<a href="#2429" id="2429">2429</a>
+<a href="#2430" id="2430">2430</a>
+<a href="#2431" id="2431">2431</a>
+<a href="#2432" id="2432">2432</a>
+<a href="#2433" id="2433">2433</a>
+<a href="#2434" id="2434">2434</a>
+<a href="#2435" id="2435">2435</a>
+<a href="#2436" id="2436">2436</a>
+<a href="#2437" id="2437">2437</a>
+<a href="#2438" id="2438">2438</a>
+<a href="#2439" id="2439">2439</a>
+<a href="#2440" id="2440">2440</a>
+<a href="#2441" id="2441">2441</a>
+<a href="#2442" id="2442">2442</a>
+<a href="#2443" id="2443">2443</a>
+<a href="#2444" id="2444">2444</a>
+<a href="#2445" id="2445">2445</a>
+<a href="#2446" id="2446">2446</a>
+<a href="#2447" id="2447">2447</a>
+<a href="#2448" id="2448">2448</a>
+<a href="#2449" id="2449">2449</a>
+<a href="#2450" id="2450">2450</a>
+<a href="#2451" id="2451">2451</a>
+<a href="#2452" id="2452">2452</a>
+<a href="#2453" id="2453">2453</a>
+<a href="#2454" id="2454">2454</a>
+<a href="#2455" id="2455">2455</a>
+<a href="#2456" id="2456">2456</a>
+<a href="#2457" id="2457">2457</a>
+<a href="#2458" id="2458">2458</a>
+<a href="#2459" id="2459">2459</a>
+<a href="#2460" id="2460">2460</a>
+<a href="#2461" id="2461">2461</a>
+<a href="#2462" id="2462">2462</a>
+<a href="#2463" id="2463">2463</a>
+<a href="#2464" id="2464">2464</a>
+<a href="#2465" id="2465">2465</a>
+<a href="#2466" id="2466">2466</a>
+<a href="#2467" id="2467">2467</a>
+<a href="#2468" id="2468">2468</a>
+<a href="#2469" id="2469">2469</a>
+<a href="#2470" id="2470">2470</a>
+<a href="#2471" id="2471">2471</a>
+<a href="#2472" id="2472">2472</a>
+<a href="#2473" id="2473">2473</a>
+<a href="#2474" id="2474">2474</a>
+<a href="#2475" id="2475">2475</a>
+<a href="#2476" id="2476">2476</a>
+<a href="#2477" id="2477">2477</a>
+<a href="#2478" id="2478">2478</a>
+<a href="#2479" id="2479">2479</a>
+<a href="#2480" id="2480">2480</a>
+<a href="#2481" id="2481">2481</a>
+<a href="#2482" id="2482">2482</a>
+<a href="#2483" id="2483">2483</a>
+<a href="#2484" id="2484">2484</a>
+<a href="#2485" id="2485">2485</a>
+<a href="#2486" id="2486">2486</a>
+<a href="#2487" id="2487">2487</a>
+<a href="#2488" id="2488">2488</a>
+<a href="#2489" id="2489">2489</a>
+<a href="#2490" id="2490">2490</a>
+<a href="#2491" id="2491">2491</a>
+<a href="#2492" id="2492">2492</a>
+<a href="#2493" id="2493">2493</a>
+<a href="#2494" id="2494">2494</a>
+<a href="#2495" id="2495">2495</a>
+<a href="#2496" id="2496">2496</a>
+<a href="#2497" id="2497">2497</a>
+<a href="#2498" id="2498">2498</a>
+<a href="#2499" id="2499">2499</a>
+<a href="#2500" id="2500">2500</a>
+<a href="#2501" id="2501">2501</a>
+<a href="#2502" id="2502">2502</a>
+<a href="#2503" id="2503">2503</a>
+<a href="#2504" id="2504">2504</a>
+<a href="#2505" id="2505">2505</a>
+<a href="#2506" id="2506">2506</a>
+<a href="#2507" id="2507">2507</a>
+<a href="#2508" id="2508">2508</a>
+<a href="#2509" id="2509">2509</a>
+<a href="#2510" id="2510">2510</a>
+<a href="#2511" id="2511">2511</a>
+<a href="#2512" id="2512">2512</a>
+<a href="#2513" id="2513">2513</a>
+<a href="#2514" id="2514">2514</a>
+<a href="#2515" id="2515">2515</a>
+<a href="#2516" id="2516">2516</a>
+<a href="#2517" id="2517">2517</a>
+<a href="#2518" id="2518">2518</a>
+<a href="#2519" id="2519">2519</a>
+<a href="#2520" id="2520">2520</a>
+<a href="#2521" id="2521">2521</a>
+<a href="#2522" id="2522">2522</a>
+<a href="#2523" id="2523">2523</a>
+<a href="#2524" id="2524">2524</a>
+<a href="#2525" id="2525">2525</a>
+<a href="#2526" id="2526">2526</a>
+<a href="#2527" id="2527">2527</a>
+<a href="#2528" id="2528">2528</a>
+<a href="#2529" id="2529">2529</a>
+<a href="#2530" id="2530">2530</a>
+<a href="#2531" id="2531">2531</a>
+<a href="#2532" id="2532">2532</a>
+<a href="#2533" id="2533">2533</a>
+<a href="#2534" id="2534">2534</a>
+<a href="#2535" id="2535">2535</a>
+<a href="#2536" id="2536">2536</a>
+<a href="#2537" id="2537">2537</a>
+<a href="#2538" id="2538">2538</a>
+<a href="#2539" id="2539">2539</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -2615,6 +2770,7 @@
quote: <span class="prelude-ty">Option</span><u8>,
terminator: <span class="prelude-ty">Option</span><u8>,
null_regex: NullRegex,
+ truncated_rows: bool,
}
<span class="kw">impl </span>Format {
@@ -2649,6 +2805,17 @@
<span class="self">self
</span>}
+ <span class="doccomment">/// Whether to allow truncated rows when parsing.
+ ///
+ /// By default this is set to `false` and will error if the CSV rows have different lengths.
+ /// When set to true then it will allow records with less than the expected number of columns
+ /// and fill the missing columns with nulls. If the record's schema is not nullable, then it
+ /// will still return an error.
+ </span><span class="kw">pub fn </span>with_truncated_rows(<span class="kw-2">mut </span><span class="self">self</span>, allow: bool) -> <span class="self">Self </span>{
+ <span class="self">self</span>.truncated_rows = allow;
+ <span class="self">self
+ </span>}
+
<span class="doccomment">/// Infer schema of CSV records from the provided `reader`
///
/// If `max_records` is `None`, all records will be read, otherwise up to `max_records`
@@ -2713,6 +2880,7 @@
</span><span class="kw">fn </span>build_reader<R: Read>(<span class="kw-2">&</span><span class="self">self</span>, reader: R) -> csv::Reader<R> {
<span class="kw">let </span><span class="kw-2">mut </span>builder = csv::ReaderBuilder::new();
builder.has_headers(<span class="self">self</span>.header);
+ builder.flexible(<span class="self">self</span>.truncated_rows);
<span class="kw">if let </span><span class="prelude-val">Some</span>(c) = <span class="self">self</span>.delimiter {
builder.delimiter(c);
@@ -3505,6 +3673,17 @@
<span class="self">self
</span>}
+ <span class="doccomment">/// Whether to allow truncated rows when parsing.
+ ///
+ /// By default this is set to `false` and will error if the CSV rows have different lengths.
+ /// When set to true then it will allow records with less than the expected number of columns
+ /// and fill the missing columns with nulls. If the record's schema is not nullable, then it
+ /// will still return an error.
+ </span><span class="kw">pub fn </span>with_truncated_rows(<span class="kw-2">mut </span><span class="self">self</span>, allow: bool) -> <span class="self">Self </span>{
+ <span class="self">self</span>.format.truncated_rows = allow;
+ <span class="self">self
+ </span>}
+
<span class="doccomment">/// Create a new `Reader` from a non-buffered reader
///
/// If `R: BufRead` consider using [`Self::build_buffered`] to avoid unnecessary additional
@@ -3524,7 +3703,11 @@
<span class="doccomment">/// Builds a decoder that can be used to decode CSV from an arbitrary byte stream
</span><span class="kw">pub fn </span>build_decoder(<span class="self">self</span>) -> Decoder {
<span class="kw">let </span>delimiter = <span class="self">self</span>.format.build_parser();
- <span class="kw">let </span>record_decoder = RecordDecoder::new(delimiter, <span class="self">self</span>.schema.fields().len());
+ <span class="kw">let </span>record_decoder = RecordDecoder::new(
+ delimiter,
+ <span class="self">self</span>.schema.fields().len(),
+ <span class="self">self</span>.format.truncated_rows,
+ );
<span class="kw">let </span>header = <span class="self">self</span>.format.header <span class="kw">as </span>usize;
@@ -4548,6 +4731,133 @@
<span class="macro">assert!</span>(c.is_null(<span class="number">3</span>));
}
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>test_truncated_rows() {
+ <span class="kw">let </span>data = <span class="string">"a,b,c\n1,2,3\n4,5\n\n6,7,8"</span>;
+ <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[
+ Field::new(<span class="string">"a"</span>, DataType::Int32, <span class="bool-val">true</span>),
+ Field::new(<span class="string">"b"</span>, DataType::Int32, <span class="bool-val">true</span>),
+ Field::new(<span class="string">"c"</span>, DataType::Int32, <span class="bool-val">true</span>),
+ ]));
+
+ <span class="kw">let </span>reader = ReaderBuilder::new(schema.clone())
+ .with_header(<span class="bool-val">true</span>)
+ .with_truncated_rows(<span class="bool-val">true</span>)
+ .build(Cursor::new(data))
+ .unwrap();
+
+ <span class="kw">let </span>batches = reader.collect::<<span class="prelude-ty">Result</span><Vec<<span class="kw">_</span>>, <span class="kw">_</span>>>();
+ <span class="macro">assert!</span>(batches.is_ok());
+ <span class="kw">let </span>batch = batches.unwrap().into_iter().next().unwrap();
+ <span class="comment">// Empty rows are skipped by the underlying csv parser
+ </span><span class="macro">assert_eq!</span>(batch.num_rows(), <span class="number">3</span>);
+
+ <span class="kw">let </span>reader = ReaderBuilder::new(schema.clone())
+ .with_header(<span class="bool-val">true</span>)
+ .with_truncated_rows(<span class="bool-val">false</span>)
+ .build(Cursor::new(data))
+ .unwrap();
+
+ <span class="kw">let </span>batches = reader.collect::<<span class="prelude-ty">Result</span><Vec<<span class="kw">_</span>>, <span class="kw">_</span>>>();
+ <span class="macro">assert!</span>(<span class="kw">match </span>batches {
+ <span class="prelude-val">Err</span>(ArrowError::CsvError(e)) => e.to_string().contains(<span class="string">"incorrect number of fields"</span>),
+ <span class="kw">_ </span>=> <span class="bool-val">false</span>,
+ });
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>test_truncated_rows_csv() {
+ <span class="kw">let </span>file = File::open(<span class="string">"test/data/truncated_rows.csv"</span>).unwrap();
+ <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[
+ Field::new(<span class="string">"Name"</span>, DataType::Utf8, <span class="bool-val">true</span>),
+ Field::new(<span class="string">"Age"</span>, DataType::UInt32, <span class="bool-val">true</span>),
+ Field::new(<span class="string">"Occupation"</span>, DataType::Utf8, <span class="bool-val">true</span>),
+ Field::new(<span class="string">"DOB"</span>, DataType::Date32, <span class="bool-val">true</span>),
+ ]));
+ <span class="kw">let </span>reader = ReaderBuilder::new(schema.clone())
+ .with_header(<span class="bool-val">true</span>)
+ .with_batch_size(<span class="number">24</span>)
+ .with_truncated_rows(<span class="bool-val">true</span>);
+ <span class="kw">let </span>csv = reader.build(file).unwrap();
+ <span class="kw">let </span>batches = csv.collect::<<span class="prelude-ty">Result</span><Vec<<span class="kw">_</span>>, <span class="kw">_</span>>>().unwrap();
+
+ <span class="macro">assert_eq!</span>(batches.len(), <span class="number">1</span>);
+ <span class="kw">let </span>batch = <span class="kw-2">&</span>batches[<span class="number">0</span>];
+ <span class="macro">assert_eq!</span>(batch.num_rows(), <span class="number">6</span>);
+ <span class="macro">assert_eq!</span>(batch.num_columns(), <span class="number">4</span>);
+ <span class="kw">let </span>name = batch
+ .column(<span class="number">0</span>)
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .unwrap();
+ <span class="kw">let </span>age = batch
+ .column(<span class="number">1</span>)
+ .as_any()
+ .downcast_ref::<UInt32Array>()
+ .unwrap();
+ <span class="kw">let </span>occupation = batch
+ .column(<span class="number">2</span>)
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .unwrap();
+ <span class="kw">let </span>dob = batch
+ .column(<span class="number">3</span>)
+ .as_any()
+ .downcast_ref::<Date32Array>()
+ .unwrap();
+
+ <span class="macro">assert_eq!</span>(name.value(<span class="number">0</span>), <span class="string">"A1"</span>);
+ <span class="macro">assert_eq!</span>(name.value(<span class="number">1</span>), <span class="string">"B2"</span>);
+ <span class="macro">assert!</span>(name.is_null(<span class="number">2</span>));
+ <span class="macro">assert_eq!</span>(name.value(<span class="number">3</span>), <span class="string">"C3"</span>);
+ <span class="macro">assert_eq!</span>(name.value(<span class="number">4</span>), <span class="string">"D4"</span>);
+ <span class="macro">assert_eq!</span>(name.value(<span class="number">5</span>), <span class="string">"E5"</span>);
+
+ <span class="macro">assert_eq!</span>(age.value(<span class="number">0</span>), <span class="number">34</span>);
+ <span class="macro">assert_eq!</span>(age.value(<span class="number">1</span>), <span class="number">29</span>);
+ <span class="macro">assert!</span>(age.is_null(<span class="number">2</span>));
+ <span class="macro">assert_eq!</span>(age.value(<span class="number">3</span>), <span class="number">45</span>);
+ <span class="macro">assert!</span>(age.is_null(<span class="number">4</span>));
+ <span class="macro">assert_eq!</span>(age.value(<span class="number">5</span>), <span class="number">31</span>);
+
+ <span class="macro">assert_eq!</span>(occupation.value(<span class="number">0</span>), <span class="string">"Engineer"</span>);
+ <span class="macro">assert_eq!</span>(occupation.value(<span class="number">1</span>), <span class="string">"Doctor"</span>);
+ <span class="macro">assert!</span>(occupation.is_null(<span class="number">2</span>));
+ <span class="macro">assert_eq!</span>(occupation.value(<span class="number">3</span>), <span class="string">"Artist"</span>);
+ <span class="macro">assert!</span>(occupation.is_null(<span class="number">4</span>));
+ <span class="macro">assert!</span>(occupation.is_null(<span class="number">5</span>));
+
+ <span class="macro">assert_eq!</span>(dob.value(<span class="number">0</span>), <span class="number">5675</span>);
+ <span class="macro">assert!</span>(dob.is_null(<span class="number">1</span>));
+ <span class="macro">assert!</span>(dob.is_null(<span class="number">2</span>));
+ <span class="macro">assert_eq!</span>(dob.value(<span class="number">3</span>), -<span class="number">1858</span>);
+ <span class="macro">assert!</span>(dob.is_null(<span class="number">4</span>));
+ <span class="macro">assert!</span>(dob.is_null(<span class="number">5</span>));
+ }
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>test_truncated_rows_not_nullable_error() {
+ <span class="kw">let </span>data = <span class="string">"a,b,c\n1,2,3\n4,5"</span>;
+ <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[
+ Field::new(<span class="string">"a"</span>, DataType::Int32, <span class="bool-val">false</span>),
+ Field::new(<span class="string">"b"</span>, DataType::Int32, <span class="bool-val">false</span>),
+ Field::new(<span class="string">"c"</span>, DataType::Int32, <span class="bool-val">false</span>),
+ ]));
+
+ <span class="kw">let </span>reader = ReaderBuilder::new(schema.clone())
+ .with_header(<span class="bool-val">true</span>)
+ .with_truncated_rows(<span class="bool-val">true</span>)
+ .build(Cursor::new(data))
+ .unwrap();
+
+ <span class="kw">let </span>batches = reader.collect::<<span class="prelude-ty">Result</span><Vec<<span class="kw">_</span>>, <span class="kw">_</span>>>();
+ <span class="macro">assert!</span>(<span class="kw">match </span>batches {
+ <span class="prelude-val">Err</span>(ArrowError::InvalidArgumentError(e)) =>
+ e.to_string().contains(<span class="string">"contains null values"</span>),
+ <span class="kw">_ </span>=> <span class="bool-val">false</span>,
+ });
+ }
+
<span class="attr">#[test]
</span><span class="kw">fn </span>test_buffered() {
<span class="kw">let </span>tests = [
diff --git a/src/arrow_csv/reader/records.rs.html b/src/arrow_csv/reader/records.rs.html
index 172064f8dfb..cc08f77cf21 100644
--- a/src/arrow_csv/reader/records.rs.html
+++ b/src/arrow_csv/reader/records.rs.html
@@ -360,6 +360,31 @@
<a href="#360" id="360">360</a>
<a href="#361" id="361">361</a>
<a href="#362" id="362">362</a>
+<a href="#363" id="363">363</a>
+<a href="#364" id="364">364</a>
+<a href="#365" id="365">365</a>
+<a href="#366" id="366">366</a>
+<a href="#367" id="367">367</a>
+<a href="#368" id="368">368</a>
+<a href="#369" id="369">369</a>
+<a href="#370" id="370">370</a>
+<a href="#371" id="371">371</a>
+<a href="#372" id="372">372</a>
+<a href="#373" id="373">373</a>
+<a href="#374" id="374">374</a>
+<a href="#375" id="375">375</a>
+<a href="#376" id="376">376</a>
+<a href="#377" id="377">377</a>
+<a href="#378" id="378">378</a>
+<a href="#379" id="379">379</a>
+<a href="#380" id="380">380</a>
+<a href="#381" id="381">381</a>
+<a href="#382" id="382">382</a>
+<a href="#383" id="383">383</a>
+<a href="#384" id="384">384</a>
+<a href="#385" id="385">385</a>
+<a href="#386" id="386">386</a>
+<a href="#387" id="387">387</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -418,10 +443,16 @@
///
/// We track this independently of Vec to avoid re-zeroing memory
</span>data_len: usize,
+
+ <span class="doccomment">/// Whether rows with less than expected columns are considered valid
+ ///
+ /// Default value is false
+ /// When enabled fills in missing columns with null
+ </span>truncated_rows: bool,
}
<span class="kw">impl </span>RecordDecoder {
- <span class="kw">pub fn </span>new(delimiter: Reader, num_columns: usize) -> <span class="self">Self </span>{
+ <span class="kw">pub fn </span>new(delimiter: Reader, num_columns: usize, truncated_rows: bool) -> <span class="self">Self </span>{
<span class="self">Self </span>{
delimiter,
num_columns,
@@ -432,6 +463,7 @@
data_len: <span class="number">0</span>,
data: <span class="macro">vec!</span>[],
num_rows: <span class="number">0</span>,
+ truncated_rows,
}
}
@@ -489,10 +521,19 @@
}
ReadRecordResult::Record => {
<span class="kw">if </span><span class="self">self</span>.current_field != <span class="self">self</span>.num_columns {
- <span class="kw">return </span><span class="prelude-val">Err</span>(ArrowError::CsvError(<span class="macro">format!</span>(
- <span class="string">"incorrect number of fields for line {}, expected {} got {}"</span>,
- <span class="self">self</span>.line_number, <span class="self">self</span>.num_columns, <span class="self">self</span>.current_field
- )));
+ <span class="kw">if </span><span class="self">self</span>.truncated_rows && <span class="self">self</span>.current_field < <span class="self">self</span>.num_columns {
+ <span class="comment">// If the number of fields is less than expected, pad with nulls
+ </span><span class="kw">let </span>fill_count = <span class="self">self</span>.num_columns - <span class="self">self</span>.current_field;
+ <span class="kw">let </span>fill_value = <span class="self">self</span>.offsets[<span class="self">self</span>.offsets_len - <span class="number">1</span>];
+ <span class="self">self</span>.offsets[<span class="self">self</span>.offsets_len..<span class="self">self</span>.offsets_len + fill_count]
+ .fill(fill_value);
+ <span class="self">self</span>.offsets_len += fill_count;
+ } <span class="kw">else </span>{
+ <span class="kw">return </span><span class="prelude-val">Err</span>(ArrowError::CsvError(<span class="macro">format!</span>(
+ <span class="string">"incorrect number of fields for line {}, expected {} got {}"</span>,
+ <span class="self">self</span>.line_number, <span class="self">self</span>.num_columns, <span class="self">self</span>.current_field
+ )));
+ }
}
read += <span class="number">1</span>;
<span class="self">self</span>.current_field = <span class="number">0</span>;
@@ -661,7 +702,7 @@
.into_iter();
<span class="kw">let </span><span class="kw-2">mut </span>reader = BufReader::with_capacity(<span class="number">3</span>, Cursor::new(csv.as_bytes()));
- <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">3</span>);
+ <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">3</span>, <span class="bool-val">false</span>);
<span class="kw">loop </span>{
<span class="kw">let </span>to_read = <span class="number">3</span>;
@@ -695,7 +736,7 @@
<span class="attr">#[test]
</span><span class="kw">fn </span>test_invalid_fields() {
<span class="kw">let </span>csv = <span class="string">"a,b\nb,c\na\n"</span>;
- <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">2</span>);
+ <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">2</span>, <span class="bool-val">false</span>);
<span class="kw">let </span>err = decoder.decode(csv.as_bytes(), <span class="number">4</span>).unwrap_err().to_string();
<span class="kw">let </span>expected = <span class="string">"Csv error: incorrect number of fields for line 3, expected 2 got 1"</span>;
@@ -703,7 +744,7 @@
<span class="macro">assert_eq!</span>(err, expected);
<span class="comment">// Test with initial skip
- </span><span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">2</span>);
+ </span><span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">2</span>, <span class="bool-val">false</span>);
<span class="kw">let </span>(skipped, bytes) = decoder.decode(csv.as_bytes(), <span class="number">1</span>).unwrap();
<span class="macro">assert_eq!</span>(skipped, <span class="number">1</span>);
decoder.clear();
@@ -716,10 +757,19 @@
<span class="attr">#[test]
</span><span class="kw">fn </span>test_skip_insufficient_rows() {
<span class="kw">let </span>csv = <span class="string">"a\nv\n"</span>;
- <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">1</span>);
+ <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">1</span>, <span class="bool-val">false</span>);
<span class="kw">let </span>(read, bytes) = decoder.decode(csv.as_bytes(), <span class="number">3</span>).unwrap();
<span class="macro">assert_eq!</span>(read, <span class="number">2</span>);
<span class="macro">assert_eq!</span>(bytes, csv.len());
}
+
+ <span class="attr">#[test]
+ </span><span class="kw">fn </span>test_truncated_rows() {
+ <span class="kw">let </span>csv = <span class="string">"a,b\nv\n,1\n,2\n,3\n"</span>;
+ <span class="kw">let </span><span class="kw-2">mut </span>decoder = RecordDecoder::new(Reader::new(), <span class="number">2</span>, <span class="bool-val">true</span>);
+ <span class="kw">let </span>(read, bytes) = decoder.decode(csv.as_bytes(), <span class="number">5</span>).unwrap();
+ <span class="macro">assert_eq!</span>(read, <span class="number">5</span>);
+ <span class="macro">assert_eq!</span>(bytes, csv.len());
+ }
}
</code></pre></div></section></main></body></html>
\ No newline at end of file
diff --git a/type.impl/arrow_csv/reader/struct.BufReader.js b/type.impl/arrow_csv/reader/struct.BufReader.js
index 66f41ee305f..7b4afeade7e 100644
--- a/type.impl/arrow_csv/reader/struct.BufReader.js
+++ b/type.impl/arrow_csv/reader/struct.BufReader.js
@@ -1,3 +1,3 @@
(function() {var type_impls = {
-"arrow_csv":[["<details class=\"toggle implementors-toggle\" open><summary><section id=\"impl-BufReader%3CR%3E\" class=\"impl\"><a class=\"src rightside\" href=\"src/arrow_csv/reader/mod.rs.html#486-504\">source</a><a href=\"#impl-BufReader%3CR%3E\" class=\"anchor\">§</a><h3 class=\"code-header\">impl<R: <a class=\"trait\" href=\"https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html\" title=\"trait std::io::BufRead\">BufRead</a>> <a class=\"struct\" href=\"arrow_csv/reader/st [...]
+"arrow_csv":[["<details class=\"toggle implementors-toggle\" open><summary><section id=\"impl-BufReader%3CR%3E\" class=\"impl\"><a class=\"src rightside\" href=\"src/arrow_csv/reader/mod.rs.html#499-517\">source</a><a href=\"#impl-BufReader%3CR%3E\" class=\"anchor\">§</a><h3 class=\"code-header\">impl<R: <a class=\"trait\" href=\"https://doc.rust-lang.org/nightly/std/io/trait.BufRead.html\" title=\"trait std::io::BufRead\">BufRead</a>> <a class=\"struct\" href=\"arrow_csv/reader/st [...]
};if (window.register_type_impls) {window.register_type_impls(type_impls);} else {window.pending_type_impls = type_impls;}})()
\ No newline at end of file