You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/02/08 03:14:36 UTC
[arrow] branch master updated: ARROW-4476: [Rust] [DataFusion]
Update README to cover DataFusion and new testing git submodule
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f3ee51f ARROW-4476: [Rust] [DataFusion] Update README to cover DataFusion and new testing git submodule
f3ee51f is described below
commit f3ee51f7e54326b0bd0bf0f2b6e5c8ab52cd7b02
Author: Andy Grove <an...@gmail.com>
AuthorDate: Thu Feb 7 21:14:25 2019 -0600
ARROW-4476: [Rust] [DataFusion] Update README to cover DataFusion and new testing git submodule
Author: Andy Grove <an...@gmail.com>
Closes #3558 from andygrove/ARROW-4476 and squashes the following commits:
517373678 <Andy Grove> update example code in README to match the example that is shipped
834f4dafe <Andy Grove> Update README to cover DataFusion and new testing git submodule
---
rust/README.md | 26 ++++++++++++++++++-------
rust/datafusion/README.md | 49 ++++++++++++++++++++++++++---------------------
2 files changed, 46 insertions(+), 29 deletions(-)
diff --git a/rust/README.md b/rust/README.md
index 8fe7885..9f3c9f1 100644
--- a/rust/README.md
+++ b/rust/README.md
@@ -21,21 +21,33 @@
## The Rust implementation of Arrow consists of the following crates
-- Arrow [(README)](arrow/README.md)
-- Parquet [(README)](parquet/README.md)
+| Crate | Description | Documentation |
+|-----------|-------------|---------------|
+|Arrow | Core functionality (memory layout, array builders, low level computations) | [(README)](arrow/README.md) |
+|Parquet | Parquet support | [(README)](parquet/README.md) |
+|DataFusion | In-memory query engine with SQL support | [(README)](datafusion/README.md) |
-## Run Tests
+## Prerequisites
-Parquet support in Arrow requires data to test against, this data is in a
-git submodule. To pull down this data run the following:
+Before running tests and examples it is necessary to set up the local development enviroment.
+
+### Git Submodules
+
+The tests rely on test data that is contained in git submodules.
+
+To pull down this data run the following:
```bash
git submodule update --init
```
-The data can then be found in `cpp/submodules/parquet_testing/data`.
+This populates data in two git submodules:
+
+- `cpp/submodules/parquet_testing/data` (sourced from https://github.com/apache/parquet-testing.git)
+- `testing` (sourced from https://github.com/apache/arrow-testing)
+
Create a new environment variable called `PARQUET_TEST_DATA` to point
-to this location and then `cargo test` as usual.
+to `cpp/submodules/parquet_testing/data` and then `cargo test` as usual.
## Code Formatting
diff --git a/rust/datafusion/README.md b/rust/datafusion/README.md
index eade95e..e837483 100644
--- a/rust/datafusion/README.md
+++ b/rust/datafusion/README.md
@@ -34,59 +34,64 @@ fn main() {
// define schema for data source (csv file)
let schema = Arc::new(Schema::new(vec![
- Field::new("city", DataType::Utf8, false),
- Field::new("lat", DataType::Float64, false),
- Field::new("lng", DataType::Float64, false),
+ Field::new("c1", DataType::Utf8, false),
+ Field::new("c2", DataType::UInt32, false),
+ Field::new("c3", DataType::Int8, false),
+ Field::new("c4", DataType::Int16, false),
+ Field::new("c5", DataType::Int32, false),
+ Field::new("c6", DataType::Int64, false),
+ Field::new("c7", DataType::UInt8, false),
+ Field::new("c8", DataType::UInt16, false),
+ Field::new("c9", DataType::UInt32, false),
+ Field::new("c10", DataType::UInt64, false),
+ Field::new("c11", DataType::Float32, false),
+ Field::new("c12", DataType::Float64, false),
+ Field::new("c13", DataType::Utf8, false),
]));
// register csv file with the execution context
- let csv_datasource = CsvDataSource::new("../../testing/data/csv/uk_cities.csv", schema.clone(), 1024);
- ctx.register_datasource("cities", Rc::new(RefCell::new(csv_datasource)));
-
- // simple projection and selection
- let sql = "SELECT city, lat, lng FROM cities WHERE lat > 51.0 AND lat < 53";
+ let csv_datasource = CsvDataSource::new(
+ "../../testing/data/csv/aggregate_test_100.csv",
+ schema.clone(),
+ 1024,
+ );
+ ctx.register_datasource("aggregate_test_100", Rc::new(RefCell::new(csv_datasource)));
// execute the query
+ let sql = "SELECT c1, MIN(c12), MAX(c12) FROM aggregate_test_100 WHERE c11 > 0.1 AND c11 < 0.9 GROUP BY c1";
let relation = ctx.sql(&sql).unwrap();
-
- // display the relation
let mut results = relation.borrow_mut();
+ // iterate over result batches
while let Some(batch) = results.next().unwrap() {
-
println!(
"RecordBatch has {} rows and {} columns",
batch.num_rows(),
batch.num_columns()
);
- let city = batch
+ let c1 = batch
.column(0)
.as_any()
.downcast_ref::<BinaryArray>()
.unwrap();
- let lat = batch
+ let min = batch
.column(1)
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
- let lng = batch
+ let max = batch
.column(2)
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
for i in 0..batch.num_rows() {
- let city_name: String = String::from_utf8(city.get_value(i).to_vec()).unwrap();
-
- println!(
- "City: {}, Latitude: {}, Longitude: {}",
- city_name,
- lat.value(i),
- lng.value(i),
- );
+ let c1_value: String = String::from_utf8(c1.value(i).to_vec()).unwrap();
+
+ println!("{}, Min: {}, Max: {}", c1_value, min.value(i), max.value(i),);
}
}
}