You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/01/28 09:25:53 UTC

[arrow] branch master updated: ARROW-4376: [Rust] Implement from_buf_reader for csv::Reader

This is an automated email from the ASF dual-hosted git repository.

kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 815492c  ARROW-4376: [Rust] Implement from_buf_reader for csv::Reader
815492c is described below

commit 815492c0ffa84118bb826d68f3684e939aeea325
Author: Paddy Horan <pa...@hotmail.com>
AuthorDate: Mon Jan 28 10:25:23 2019 +0100

    ARROW-4376: [Rust] Implement from_buf_reader for csv::Reader
    
    Update to allow the csv reader to work with anything that implements the `Read` trait.
    
    Also adds a new constructor `from_buf_reader`.  Although 90% of use cases will just read a csv file on disk there are some situations where to need to pre-process or otherwise adjust the data before sending it to the csv reader.
    
    Author: Paddy Horan <pa...@hotmail.com>
    
    Closes #3508 from paddyhoran/from_buf_reader and squashes the following commits:
    
    2fe08693 <Paddy Horan> Added `from_buf_reader` and made `Reader` generic over `Read`
---
 rust/arrow/src/csv/reader.rs | 69 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 58 insertions(+), 11 deletions(-)

diff --git a/rust/arrow/src/csv/reader.rs b/rust/arrow/src/csv/reader.rs
index e16d7c8..4582171 100644
--- a/rust/arrow/src/csv/reader.rs
+++ b/rust/arrow/src/csv/reader.rs
@@ -44,7 +44,7 @@ use lazy_static::lazy_static;
 use regex::{Regex, RegexBuilder};
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::{BufReader, Seek, SeekFrom};
+use std::io::{BufReader, Read, Seek, SeekFrom};
 use std::sync::Arc;
 
 use csv as csv_crate;
@@ -183,23 +183,43 @@ fn infer_file_schema(
 }
 
 /// CSV file reader
-pub struct Reader {
+pub struct Reader<R: Read> {
     /// Explicit schema for the CSV file
     schema: Arc<Schema>,
     /// Optional projection for which columns to load (zero-based column indices)
     projection: Option<Vec<usize>>,
     /// File reader
-    record_iter: StringRecordsIntoIter<BufReader<File>>,
+    record_iter: StringRecordsIntoIter<BufReader<R>>,
     /// Batch size (number of records to load each time)
     batch_size: usize,
 }
 
-impl Reader {
-    /// Create a new CsvReader
+impl<R: Read> Reader<R> {
+    /// Create a new CsvReader from any value that implements the `Read` trait.
     ///
-    /// To customise the Reader, such as to enable schema inference, use `ReaderBuilder`
+    /// If reading a `File` you can customise the Reader, such as to enable schema inference, use `ReaderBuilder`.
     pub fn new(
-        file: File,
+        reader: R,
+        schema: Arc<Schema>,
+        has_headers: bool,
+        batch_size: usize,
+        projection: Option<Vec<usize>>,
+    ) -> Self {
+        Self::from_buf_reader(
+            BufReader::new(reader),
+            schema,
+            has_headers,
+            batch_size,
+            projection,
+        )
+    }
+
+    /// Create a new CsvReader from a `BufReader<R: Read>
+    ///
+    /// This constructor allows you more flexibility in what records are processed by the csv
+    /// reader.
+    pub fn from_buf_reader(
+        buf_reader: BufReader<R>,
         schema: Arc<Schema>,
         has_headers: bool,
         batch_size: usize,
@@ -207,9 +227,9 @@ impl Reader {
     ) -> Self {
         let csv_reader = csv::ReaderBuilder::new()
             .has_headers(has_headers)
-            .from_reader(BufReader::new(file));
+            .from_reader(buf_reader);
         let record_iter = csv_reader.into_records();
-        Reader {
+        Self {
             schema,
             projection,
             record_iter,
@@ -390,7 +410,7 @@ impl ReaderBuilder {
     /// use arrow::csv;
     /// use std::fs::File;
     ///
-    /// fn example() -> csv::Reader {
+    /// fn example() -> csv::Reader<File> {
     ///     let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();
     ///
     ///     // create a builder, inferring the schema with the first 100 records
@@ -444,7 +464,7 @@ impl ReaderBuilder {
     }
 
     /// Create a new `Reader` from the `ReaderBuilder`
-    pub fn build(self, file: File) -> Result<Reader> {
+    pub fn build(self, file: File) -> Result<Reader<File>> {
         // check if schema should be inferred
         let schema = match self.schema {
             Some(schema) => schema,
@@ -477,6 +497,8 @@ impl ReaderBuilder {
 mod tests {
     use super::*;
 
+    use std::io::Cursor;
+
     use crate::array::*;
     use crate::datatypes::Field;
 
@@ -516,6 +538,31 @@ mod tests {
     }
 
     #[test]
+    fn test_csv_from_buf_reader() {
+        let schema = Schema::new(vec![
+            Field::new("city", DataType::Utf8, false),
+            Field::new("lat", DataType::Float64, false),
+            Field::new("lng", DataType::Float64, false),
+        ]);
+
+        let file_with_headers = File::open("test/data/uk_cities_with_headers.csv").unwrap();
+        let file_without_headers = File::open("test/data/uk_cities.csv").unwrap();
+        let both_files = file_with_headers
+            .chain(Cursor::new("\n".to_string()))
+            .chain(file_without_headers);
+        let mut csv = Reader::from_buf_reader(
+            BufReader::new(both_files),
+            Arc::new(schema),
+            true,
+            1024,
+            None,
+        );
+        let batch = csv.next().unwrap().unwrap();
+        assert_eq!(74, batch.num_rows());
+        assert_eq!(3, batch.num_columns());
+    }
+
+    #[test]
     fn test_csv_with_schema_inference() {
         let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();