You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/01/28 09:25:53 UTC
[arrow] branch master updated: ARROW-4376: [Rust] Implement
from_buf_reader for csv::Reader
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 815492c ARROW-4376: [Rust] Implement from_buf_reader for csv::Reader
815492c is described below
commit 815492c0ffa84118bb826d68f3684e939aeea325
Author: Paddy Horan <pa...@hotmail.com>
AuthorDate: Mon Jan 28 10:25:23 2019 +0100
ARROW-4376: [Rust] Implement from_buf_reader for csv::Reader
Update to allow the csv reader to work with anything that implements the `Read` trait.
Also adds a new constructor `from_buf_reader`. Although 90% of use cases will just read a csv file on disk there are some situations where to need to pre-process or otherwise adjust the data before sending it to the csv reader.
Author: Paddy Horan <pa...@hotmail.com>
Closes #3508 from paddyhoran/from_buf_reader and squashes the following commits:
2fe08693 <Paddy Horan> Added `from_buf_reader` and made `Reader` generic over `Read`
---
rust/arrow/src/csv/reader.rs | 69 +++++++++++++++++++++++++++++++++++++-------
1 file changed, 58 insertions(+), 11 deletions(-)
diff --git a/rust/arrow/src/csv/reader.rs b/rust/arrow/src/csv/reader.rs
index e16d7c8..4582171 100644
--- a/rust/arrow/src/csv/reader.rs
+++ b/rust/arrow/src/csv/reader.rs
@@ -44,7 +44,7 @@ use lazy_static::lazy_static;
use regex::{Regex, RegexBuilder};
use std::collections::HashSet;
use std::fs::File;
-use std::io::{BufReader, Seek, SeekFrom};
+use std::io::{BufReader, Read, Seek, SeekFrom};
use std::sync::Arc;
use csv as csv_crate;
@@ -183,23 +183,43 @@ fn infer_file_schema(
}
/// CSV file reader
-pub struct Reader {
+pub struct Reader<R: Read> {
/// Explicit schema for the CSV file
schema: Arc<Schema>,
/// Optional projection for which columns to load (zero-based column indices)
projection: Option<Vec<usize>>,
/// File reader
- record_iter: StringRecordsIntoIter<BufReader<File>>,
+ record_iter: StringRecordsIntoIter<BufReader<R>>,
/// Batch size (number of records to load each time)
batch_size: usize,
}
-impl Reader {
- /// Create a new CsvReader
+impl<R: Read> Reader<R> {
+ /// Create a new CsvReader from any value that implements the `Read` trait.
///
- /// To customise the Reader, such as to enable schema inference, use `ReaderBuilder`
+ /// If reading a `File` you can customise the Reader, such as to enable schema inference, use `ReaderBuilder`.
pub fn new(
- file: File,
+ reader: R,
+ schema: Arc<Schema>,
+ has_headers: bool,
+ batch_size: usize,
+ projection: Option<Vec<usize>>,
+ ) -> Self {
+ Self::from_buf_reader(
+ BufReader::new(reader),
+ schema,
+ has_headers,
+ batch_size,
+ projection,
+ )
+ }
+
+ /// Create a new CsvReader from a `BufReader<R: Read>
+ ///
+ /// This constructor allows you more flexibility in what records are processed by the csv
+ /// reader.
+ pub fn from_buf_reader(
+ buf_reader: BufReader<R>,
schema: Arc<Schema>,
has_headers: bool,
batch_size: usize,
@@ -207,9 +227,9 @@ impl Reader {
) -> Self {
let csv_reader = csv::ReaderBuilder::new()
.has_headers(has_headers)
- .from_reader(BufReader::new(file));
+ .from_reader(buf_reader);
let record_iter = csv_reader.into_records();
- Reader {
+ Self {
schema,
projection,
record_iter,
@@ -390,7 +410,7 @@ impl ReaderBuilder {
/// use arrow::csv;
/// use std::fs::File;
///
- /// fn example() -> csv::Reader {
+ /// fn example() -> csv::Reader<File> {
/// let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();
///
/// // create a builder, inferring the schema with the first 100 records
@@ -444,7 +464,7 @@ impl ReaderBuilder {
}
/// Create a new `Reader` from the `ReaderBuilder`
- pub fn build(self, file: File) -> Result<Reader> {
+ pub fn build(self, file: File) -> Result<Reader<File>> {
// check if schema should be inferred
let schema = match self.schema {
Some(schema) => schema,
@@ -477,6 +497,8 @@ impl ReaderBuilder {
mod tests {
use super::*;
+ use std::io::Cursor;
+
use crate::array::*;
use crate::datatypes::Field;
@@ -516,6 +538,31 @@ mod tests {
}
#[test]
+ fn test_csv_from_buf_reader() {
+ let schema = Schema::new(vec![
+ Field::new("city", DataType::Utf8, false),
+ Field::new("lat", DataType::Float64, false),
+ Field::new("lng", DataType::Float64, false),
+ ]);
+
+ let file_with_headers = File::open("test/data/uk_cities_with_headers.csv").unwrap();
+ let file_without_headers = File::open("test/data/uk_cities.csv").unwrap();
+ let both_files = file_with_headers
+ .chain(Cursor::new("\n".to_string()))
+ .chain(file_without_headers);
+ let mut csv = Reader::from_buf_reader(
+ BufReader::new(both_files),
+ Arc::new(schema),
+ true,
+ 1024,
+ None,
+ );
+ let batch = csv.next().unwrap().unwrap();
+ assert_eq!(74, batch.num_rows());
+ assert_eq!(3, batch.num_columns());
+ }
+
+ #[test]
fn test_csv_with_schema_inference() {
let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();