You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2022/06/09 20:53:30 UTC
[arrow-rs] branch master updated: Change to use `resolver v2`, test more feature flag combinations in CI, fix errors (#1630) (#1822)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new db41b339f Change to use `resolver v2`, test more feature flag combinations in CI, fix errors (#1630) (#1822)
db41b339f is described below
commit db41b339f99efe3b4df766a7c102e58e3ac736d1
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Thu Jun 9 21:53:25 2022 +0100
Change to use `resolver v2`, test more feature flag combinations in CI, fix errors (#1630) (#1822)
* Test more feature flag combinations in CI (#1630)
* Clippy lints
* Fix clippy fix
* Fix running examples from workspace root
* Format
* Fix arrow benchmark features
* Split up CI yaml
* Add docs
* Rework caching
* Use lockfile for cache key
Don't install unused components
---
.github/actions/setup-builder/action.yaml | 27 ++-
.github/workflows/miri.yaml | 2 +-
.github/workflows/rust.yml | 251 +++++----------------
Cargo.toml | 9 +
arrow/Cargo.toml | 15 ++
arrow/examples/read_csv.rs | 3 +-
arrow/examples/read_csv_infer_schema.rs | 6 +-
arrow/test/dependency/README.md | 21 --
arrow/test/dependency/default-features/Cargo.toml | 30 ---
arrow/test/dependency/default-features/src/main.rs | 3 -
.../test/dependency/no-default-features/Cargo.toml | 30 ---
.../dependency/no-default-features/src/main.rs | 3 -
arrow/test/dependency/simd/Cargo.toml | 30 ---
arrow/test/dependency/simd/src/main.rs | 3 -
parquet/Cargo.toml | 17 +-
parquet/README.md | 2 +-
parquet/src/errors.rs | 5 +-
parquet/src/record/api.rs | 20 +-
parquet_derive/test/dependency/README.md | 21 --
.../test/dependency/default-features/Cargo.toml | 31 ---
.../test/dependency/default-features/src/main.rs | 3 -
21 files changed, 144 insertions(+), 388 deletions(-)
diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
index 13a3008b7..0157caf8c 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -25,6 +25,31 @@ inputs:
runs:
using: "composite"
steps:
+ - name: Cache Cargo
+ uses: actions/cache@v3
+ with:
+ # these represent dependencies downloaded by cargo
+ # and thus do not depend on the OS, arch nor rust version.
+ #
+ # source https://github.com/actions/cache/blob/main/examples.md#rust---cargo
+ path: |
+ /usr/local/cargo/bin/
+ /usr/local/cargo/registry/index/
+ /usr/local/cargo/registry/cache/
+ /usr/local/cargo/git/db/
+ key: cargo-cache3-${{ hashFiles('**/Cargo.toml') }}
+ restore-keys: cargo-cache3-
+ - name: Generate lockfile
+ shell: bash
+ run: cargo fetch
+ - name: Cache Rust dependencies
+ uses: actions/cache@v3
+ with:
+ # these represent compiled steps of both dependencies and arrow
+ # and thus are specific for a particular OS, arch and rust version.
+ path: /github/home/target
+ key: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-
- name: Install Build Dependencies
shell: bash
run: |
@@ -36,4 +61,4 @@ runs:
echo "Installing ${{ inputs.rust-version }}"
rustup toolchain install ${{ inputs.rust-version }}
rustup default ${{ inputs.rust-version }}
- rustup component add rustfmt
+ echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV
diff --git a/.github/workflows/miri.yaml b/.github/workflows/miri.yaml
index 2167ed519..7feacc07d 100644
--- a/.github/workflows/miri.yaml
+++ b/.github/workflows/miri.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Rust
+name: MIRI
on:
# always trigger
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 7f5996290..9331db745 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -30,8 +30,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [stable]
+ arch: [ amd64 ]
+ rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
@@ -40,39 +40,23 @@ jobs:
RUSTFLAGS: "-C debuginfo=1"
steps:
- uses: actions/checkout@v2
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- # these represent dependencies downloaded by cargo
- # and thus do not depend on the OS, arch nor rust version.
- path: /github/home/.cargo
- key: cargo-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- # these represent compiled steps of both dependencies and arrow
- # and thus are specific for a particular OS, arch and rust version.
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}-
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Build Workspace
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
cargo build
# test the crate
linux-test:
name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
- needs: [linux-build-lib]
+ needs: [ linux-build-lib ]
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [stable]
+ arch: [ amd64 ]
+ rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
@@ -85,54 +69,52 @@ jobs:
- uses: actions/checkout@v2
with:
submodules: true
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Run tests
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
-
# run tests on all workspace members with default feature list
cargo test
-
- # Switch to arrow crate
- cd arrow
- # re-run tests on arrow crate to ensure
- # all arrays are created correctly
- cargo test --features=force_validate
- cargo test --features=prettyprint
- # run test on arrow crate with minimal set of features
- cargo test --no-default-features
+ - name: Re-run tests with all supported features
+ run: |
+ cargo test -p arrow --features=force_validate,prettyprint
+ - name: Run examples
+ run: |
+ # Test arrow examples
cargo run --example builders
cargo run --example dynamic_types
cargo run --example read_csv
cargo run --example read_csv_infer_schema
- cargo check --no-default-features
-
- # Switch to parquet crate
- cd ../parquet
- # re-run tests on parquet crate with async feature enabled
- cargo test --features=async
- cargo check --no-default-features
-
- # Switch to arrow-flight
- cd ../arrow-flight
- cargo test --features=flight-sql-experimental
- cargo check --no-default-features
+ - name: Test compilation of arrow library crate with different feature combinations
+ run: |
+ cargo check -p arrow
+ cargo check -p arrow --no-default-features
+ - name: Test compilation of arrow targets with different feature combinations
+ run: |
+ cargo check -p arrow --all-targets
+ cargo check -p arrow --no-default-features --all-targets
+ cargo check -p arrow --no-default-features --all-targets --features test_utils
+ - name: Re-run tests on arrow-flight with all features
+ run: |
+ cargo test -p arrow-flight --all-features
+ - name: Re-run tests on parquet crate with all features
+ run: |
+ cargo test -p parquet --all-features
+ - name: Test compilation of parquet library crate with different feature combinations
+ run: |
+ cargo check -p parquet
+ cargo check -p parquet --no-default-features
+ cargo check -p parquet --no-default-features --features arrow
+ - name: Test compilation of parquet targets with different feature combinations
+ run: |
+ cargo check -p parquet --all-targets
+ cargo check -p parquet --no-default-features --all-targets
+ cargo check -p parquet --no-default-features --features arrow --all-targets
+ - name: Test compilation of parquet_derive macro with different feature combinations
+ run: |
+ cargo check -p parquet_derive
# test the --features "simd" of the arrow crate. This requires nightly.
linux-test-simd:
@@ -140,8 +122,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [nightly]
+ arch: [ amd64 ]
+ rust: [ nightly ]
container:
image: ${{ matrix.arch }}/rust
env:
@@ -153,40 +135,25 @@ jobs:
- uses: actions/checkout@v2
with:
submodules: true
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- path: /github/home/.cargo
- key: cargo-nightly-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-nightly-cache3-${{ matrix.rust }}
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Run tests
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd arrow
- cargo test --features "simd"
- - name: Check new project build with simd features
+ cargo test -p arrow --features "simd"
+ - name: Check compilation with simd features
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd arrow/test/dependency/simd
- cargo check
+ cargo check -p arrow --features simd
+ cargo check -p arrow --features simd --all-targets
windows-and-macos:
name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
- os: [windows-latest, macos-latest]
- rust: [stable]
+ os: [ windows-latest, macos-latest ]
+ rust: [ stable ]
steps:
- uses: actions/checkout@v2
with:
@@ -197,7 +164,6 @@ jobs:
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- name: Run tests
shell: bash
run: |
@@ -209,12 +175,12 @@ jobs:
clippy:
name: Clippy
- needs: [linux-build-lib]
+ needs: [ linux-build-lib ]
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [stable]
+ arch: [ amd64 ]
+ rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
@@ -225,29 +191,15 @@ jobs:
- uses: actions/checkout@v2
with:
submodules: true
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Setup Clippy
run: |
- rustup component add rustfmt clippy
+ rustup component add clippy
- name: Run clippy
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
cargo clippy --features test_common --features prettyprint --features=async --all-targets --workspace -- -D warnings
check_benches:
@@ -255,8 +207,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [stable]
+ arch: [ amd64 ]
+ rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
@@ -267,27 +219,13 @@ jobs:
- uses: actions/checkout@v2
with:
submodules: true
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Check benchmarks
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cargo check --benches --workspace --features test_common,prettyprint,async,experimental
+ cargo check --benches --workspace --features test_common,prettyprint,async,experimental
lint:
name: Lint (cargo fmt)
@@ -309,8 +247,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [stable]
+ arch: [ amd64 ]
+ rust: [ stable ]
steps:
- uses: actions/checkout@v2
with:
@@ -319,7 +257,6 @@ jobs:
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
- rustup component add rustfmt clippy
- name: Cache Cargo
uses: actions/cache@v3
with:
@@ -354,8 +291,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [nightly]
+ arch: [ amd64 ]
+ rust: [ nightly ]
container:
image: ${{ matrix.arch }}/rust
env:
@@ -382,13 +319,10 @@ jobs:
run: |
rustup toolchain install ${{ matrix.rust }}
rustup override set ${{ matrix.rust }}
- rustup component add rustfmt
rustup target add wasm32-unknown-unknown
rustup target add wasm32-wasi
- name: Build arrow crate
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
cd arrow
cargo build --no-default-features --features=csv,ipc,simd --target wasm32-unknown-unknown
cargo build --no-default-features --features=csv,ipc,simd --target wasm32-wasi
@@ -399,14 +333,15 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- arch: [amd64]
- rust: [nightly]
+ arch: [ amd64 ]
+ rust: [ nightly ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
+ RUSTDOCFLAGS: "-Dwarnings"
steps:
- uses: actions/checkout@v2
with:
@@ -415,74 +350,10 @@ jobs:
run: |
apt update
apt install -y libpython3.9-dev
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- path: /github/home/.cargo
- key: cargo-nightly-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-nightly-cache3-${{ matrix.rust }}
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Run cargo doc
run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- export RUSTDOCFLAGS="-Dwarnings"
cargo doc --document-private-items --no-deps --workspace --all-features
-
-
- # test builds with various feature flag combinations outside the main workspace
- default-build:
- name: Feature Flag Builds ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable debug symbol generation to speed up CI build and keep memory down
- RUSTFLAGS: "-C debuginfo=0"
- steps:
- - uses: actions/checkout@v2
- - name: Cache Cargo
- uses: actions/cache@v3
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache3-
- - name: Cache Rust dependencies
- uses: actions/cache@v3
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{ matrix.rust }}
- - name: Arrow Build with default features
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd arrow/test/dependency/default-features
- cargo check
- - name: Arrow Build with default-features=false
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd arrow/test/dependency/no-default-features
- cargo check
- - name: Parquet Derive build with default-features
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd parquet_derive/test/dependency/default-features
- cargo check
diff --git a/Cargo.toml b/Cargo.toml
index de7d36f34..2837f028e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,6 +24,15 @@ members = [
"arrow-flight",
"integration-testing",
]
+# Enable the version 2 feature resolver, which avoids unifying features for targets that are not being built
+#
+# Critically this prevents dev-dependencies from enabling features even when not building a target that
+# uses dev-dependencies, e.g. the library crate. This in turn ensures that we can catch invalid feature
+# flag combinations that would otherwise only surface in dependent crates
+#
+# Reference - https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-resolver-version-2
+#
+resolver = "2"
# this package is excluded because it requires different compilation flags, thereby significantly changing
# how it is compiled within the workspace, causing the whole workspace to be compiled from scratch
diff --git a/arrow/Cargo.toml b/arrow/Cargo.toml
index edc04ca5f..fd60078f1 100644
--- a/arrow/Cargo.toml
+++ b/arrow/Cargo.toml
@@ -88,6 +88,7 @@ tempfile = "3"
[[bench]]
name = "aggregate_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "array_from_vec"
@@ -96,6 +97,7 @@ harness = false
[[bench]]
name = "builder"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "buffer_bit_ops"
@@ -104,6 +106,7 @@ harness = false
[[bench]]
name = "boolean_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "boolean_append_packed"
@@ -112,22 +115,27 @@ harness = false
[[bench]]
name = "arithmetic_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "cast_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "comparison_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "filter_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "take_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "length_kernel"
@@ -140,10 +148,12 @@ harness = false
[[bench]]
name = "sort_kernel"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "partition_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "csv_writer"
@@ -156,6 +166,7 @@ harness = false
[[bench]]
name = "equal"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "array_slice"
@@ -164,18 +175,22 @@ harness = false
[[bench]]
name = "concatenate_kernel"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "mutable_array"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "buffer_create"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "substring_kernels"
harness = false
+required-features = ["test_utils"]
[[bench]]
name = "array_data_validate"
diff --git a/arrow/examples/read_csv.rs b/arrow/examples/read_csv.rs
index 243d8d0f7..5ccf0c58a 100644
--- a/arrow/examples/read_csv.rs
+++ b/arrow/examples/read_csv.rs
@@ -35,7 +35,8 @@ fn main() {
Field::new("lng", DataType::Float64, false),
]);
- let file = File::open("test/data/uk_cities.csv").unwrap();
+ let path = format!("{}/test/data/uk_cities.csv", env!("CARGO_MANIFEST_DIR"));
+ let file = File::open(path).unwrap();
let mut csv =
csv::Reader::new(file, Arc::new(schema), false, None, 1024, None, None, None);
diff --git a/arrow/examples/read_csv_infer_schema.rs b/arrow/examples/read_csv_infer_schema.rs
index 11f8cfb7f..e9f5ff650 100644
--- a/arrow/examples/read_csv_infer_schema.rs
+++ b/arrow/examples/read_csv_infer_schema.rs
@@ -26,7 +26,11 @@ use std::fs::File;
fn main() {
#[cfg(feature = "csv")]
{
- let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();
+ let path = format!(
+ "{}/test/data/uk_cities_with_headers.csv",
+ env!("CARGO_MANIFEST_DIR")
+ );
+ let file = File::open(path).unwrap();
let builder = csv::ReaderBuilder::new()
.has_header(true)
.infer_schema(Some(100));
diff --git a/arrow/test/dependency/README.md b/arrow/test/dependency/README.md
deleted file mode 100644
index b618b4636..000000000
--- a/arrow/test/dependency/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-This directory contains projects that use arrow as a dependency with
-various combinations of feature flags.
diff --git a/arrow/test/dependency/default-features/Cargo.toml b/arrow/test/dependency/default-features/Cargo.toml
deleted file mode 100644
index c03aef997..000000000
--- a/arrow/test/dependency/default-features/Cargo.toml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "defeault-features"
-description = "Models a user application of arrow that uses default features of arrow"
-version = "0.1.0"
-edition = "2021"
-rust-version = "1.57"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-arrow = { path = "../../../../arrow", version = "15.0.0" }
-
-[workspace]
diff --git a/arrow/test/dependency/default-features/src/main.rs b/arrow/test/dependency/default-features/src/main.rs
deleted file mode 100644
index e7a11a969..000000000
--- a/arrow/test/dependency/default-features/src/main.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-fn main() {
- println!("Hello, world!");
-}
diff --git a/arrow/test/dependency/no-default-features/Cargo.toml b/arrow/test/dependency/no-default-features/Cargo.toml
deleted file mode 100644
index c637aa6e9..000000000
--- a/arrow/test/dependency/no-default-features/Cargo.toml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "no-default-features"
-description = "Models a user application of arrow that specifies no-default-features=true"
-version = "0.1.0"
-edition = "2021"
-rust-version = "1.57"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-arrow = { path = "../../../../arrow", version = "15.0.0", default-features = false }
-
-[workspace]
diff --git a/arrow/test/dependency/no-default-features/src/main.rs b/arrow/test/dependency/no-default-features/src/main.rs
deleted file mode 100644
index e7a11a969..000000000
--- a/arrow/test/dependency/no-default-features/src/main.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-fn main() {
- println!("Hello, world!");
-}
diff --git a/arrow/test/dependency/simd/Cargo.toml b/arrow/test/dependency/simd/Cargo.toml
deleted file mode 100644
index ecc6a93a8..000000000
--- a/arrow/test/dependency/simd/Cargo.toml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "defeault-features"
-description = "Models a user application of arrow that uses the simd feature of arrow"
-version = "0.1.0"
-edition = "2021"
-rust-version = "1.57"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-arrow = { path = "../../../../arrow", version = "15.0.0", features = ["simd"]}
-
-[workspace]
diff --git a/arrow/test/dependency/simd/src/main.rs b/arrow/test/dependency/simd/src/main.rs
deleted file mode 100644
index e7a11a969..000000000
--- a/arrow/test/dependency/simd/src/main.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-fn main() {
- println!("Hello, world!");
-}
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
index bb7a8cd10..a805ea8c4 100644
--- a/parquet/Cargo.toml
+++ b/parquet/Cargo.toml
@@ -39,18 +39,19 @@ brotli = { version = "3.3", optional = true }
flate2 = { version = "1.0", optional = true }
lz4 = { version = "1.23", optional = true }
zstd = { version = "0.11.1", optional = true, default-features = false }
-chrono = { version = "0.4", default-features = false }
+chrono = { version = "0.4", default-features = false, features = ["alloc"] }
num = "0.4"
num-bigint = "0.4"
arrow = { path = "../arrow", version = "15.0.0", optional = true, default-features = false, features = ["ipc"] }
base64 = { version = "0.13", optional = true }
clap = { version = "3", optional = true, features = ["derive", "env"] }
-serde_json = { version = "1.0", features = ["preserve_order"], optional = true }
+serde_json = { version = "1.0", optional = true }
rand = "0.8"
futures = { version = "0.3", optional = true }
tokio = { version = "1.0", optional = true, default-features = false, features = ["macros", "fs", "rt", "io-util"] }
[dev-dependencies]
+base64 = "0.13"
criterion = "0.3"
rand = "0.8"
snap = "1.0"
@@ -58,19 +59,24 @@ tempfile = "3.0"
brotli = "3.3"
flate2 = "1.0"
lz4 = "1.23"
+zstd = "0.11"
serde_json = { version = "1.0", features = ["preserve_order"] }
-arrow = { path = "../arrow", version = "15.0.0", default-features = false, features = ["test_utils", "prettyprint"] }
+arrow = { path = "../arrow", version = "15.0.0", default-features = false, features = ["ipc", "test_utils", "prettyprint"] }
[package.metadata.docs.rs]
all-features = true
[features]
default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd", "base64"]
+# Enable arrow reader/writer APIs
+arrow = ["dep:arrow", "base64"]
+# Enable CLI tools
cli = ["serde_json", "base64", "clap"]
+# Enable internal testing APIs
test_common = []
# Experimental, unstable functionality primarily used for testing
experimental = []
-# Enable async API
+# Enable async APIs
async = ["futures", "tokio"]
[[bin]]
@@ -87,11 +93,12 @@ required-features = ["cli"]
[[bench]]
name = "arrow_writer"
+required-features = ["arrow"]
harness = false
[[bench]]
name = "arrow_reader"
-required-features = ["test_common", "experimental"]
+required-features = ["arrow", "test_common", "experimental"]
harness = false
[lib]
diff --git a/parquet/README.md b/parquet/README.md
index ed8b5518b..fbb6e3e1b 100644
--- a/parquet/README.md
+++ b/parquet/README.md
@@ -27,7 +27,7 @@ See [crate documentation](https://docs.rs/parquet/latest/parquet/) for examples
## Rust Version Compatbility
-This crate is tested with the latest stable version of Rust. We do not currrently test against other, older versions of the Rust compiler.
+This crate is tested with the latest stable version of Rust. We do not currently test against other, older versions of the Rust compiler.
## Features
diff --git a/parquet/src/errors.rs b/parquet/src/errors.rs
index fcbb846f1..c2fb5bd66 100644
--- a/parquet/src/errors.rs
+++ b/parquet/src/errors.rs
@@ -17,7 +17,7 @@
//! Common Parquet errors and macros.
-use std::{cell, convert, io, result, str};
+use std::{cell, io, result, str};
#[cfg(any(feature = "arrow", test))]
use arrow::error::ArrowError;
@@ -108,7 +108,7 @@ pub type Result<T> = result::Result<T, ParquetError>;
// ----------------------------------------------------------------------
// Conversion from `ParquetError` to other types of `Error`s
-impl convert::From<ParquetError> for io::Error {
+impl From<ParquetError> for io::Error {
fn from(e: ParquetError) -> Self {
io::Error::new(io::ErrorKind::Other, e)
}
@@ -135,6 +135,7 @@ macro_rules! eof_err {
($fmt:expr, $($args:expr),*) => (ParquetError::EOF(format!($fmt, $($args),*)));
}
+#[cfg(any(feature = "arrow", test))]
macro_rules! arrow_err {
($fmt:expr) => (ParquetError::ArrowError($fmt.to_owned()));
($fmt:expr, $($args:expr),*) => (ParquetError::ArrowError(format!($fmt, $($args),*)));
diff --git a/parquet/src/record/api.rs b/parquet/src/record/api.rs
index 95b97bc95..5df21e4b0 100644
--- a/parquet/src/record/api.rs
+++ b/parquet/src/record/api.rs
@@ -27,7 +27,7 @@ use crate::data_type::{ByteArray, Decimal, Int96};
use crate::errors::{ParquetError, Result};
use crate::schema::types::ColumnDescPtr;
-#[cfg(feature = "cli")]
+#[cfg(any(feature = "cli", test))]
use serde_json::Value;
/// Macro as a shortcut to generate 'not yet implemented' panic error.
@@ -79,7 +79,7 @@ impl Row {
}
}
- #[cfg(feature = "cli")]
+ #[cfg(any(feature = "cli", test))]
pub fn to_json_value(&self) -> Value {
Value::Object(
self.fields
@@ -650,7 +650,7 @@ impl Field {
}
}
- #[cfg(feature = "cli")]
+ #[cfg(any(feature = "cli", test))]
pub fn to_json_value(&self) -> Value {
match &self {
Field::Null => Value::Null,
@@ -669,7 +669,7 @@ impl Field {
Field::Double(n) => serde_json::Number::from_f64(*n)
.map(Value::Number)
.unwrap_or(Value::Null),
- Field::Decimal(n) => Value::String(convert_decimal_to_string(&n)),
+ Field::Decimal(n) => Value::String(convert_decimal_to_string(n)),
Field::Str(s) => Value::String(s.to_owned()),
Field::Bytes(b) => Value::String(base64::encode(b.data())),
Field::Date(d) => Value::String(convert_date_to_string(*d)),
@@ -1668,7 +1668,7 @@ mod tests {
}
#[test]
- #[cfg(feature = "cli")]
+ #[cfg(any(feature = "cli", test))]
fn test_to_json_value() {
assert_eq!(Field::Null.to_json_value(), Value::Null);
assert_eq!(Field::Bool(true).to_json_value(), Value::Bool(true));
@@ -1707,21 +1707,19 @@ mod tests {
);
assert_eq!(
Field::Float(5.0).to_json_value(),
- Value::Number(serde_json::Number::from_f64(f64::from(5.0 as f32)).unwrap())
+ Value::Number(serde_json::Number::from_f64(5.0).unwrap())
);
assert_eq!(
Field::Float(5.1234).to_json_value(),
- Value::Number(
- serde_json::Number::from_f64(f64::from(5.1234 as f32)).unwrap()
- )
+ Value::Number(serde_json::Number::from_f64(5.1234_f32 as f64).unwrap())
);
assert_eq!(
Field::Double(6.0).to_json_value(),
- Value::Number(serde_json::Number::from_f64(6.0 as f64).unwrap())
+ Value::Number(serde_json::Number::from_f64(6.0).unwrap())
);
assert_eq!(
Field::Double(6.1234).to_json_value(),
- Value::Number(serde_json::Number::from_f64(6.1234 as f64).unwrap())
+ Value::Number(serde_json::Number::from_f64(6.1234).unwrap())
);
assert_eq!(
Field::Str("abc".to_string()).to_json_value(),
diff --git a/parquet_derive/test/dependency/README.md b/parquet_derive/test/dependency/README.md
deleted file mode 100644
index b618b4636..000000000
--- a/parquet_derive/test/dependency/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-This directory contains projects that use arrow as a dependency with
-various combinations of feature flags.
diff --git a/parquet_derive/test/dependency/default-features/Cargo.toml b/parquet_derive/test/dependency/default-features/Cargo.toml
deleted file mode 100644
index 7434552aa..000000000
--- a/parquet_derive/test/dependency/default-features/Cargo.toml
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "defeault-features"
-description = "Models a user application of parquet_derive that uses no additional features of arrow"
-version = "0.1.0"
-edition = "2021"
-rust-version = "1.57"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-parquet_derive = { path = "../../../../parquet_derive", version = "15.0.0" }
-
-# Keep this out of the default workspace
-[workspace]
diff --git a/parquet_derive/test/dependency/default-features/src/main.rs b/parquet_derive/test/dependency/default-features/src/main.rs
deleted file mode 100644
index e7a11a969..000000000
--- a/parquet_derive/test/dependency/default-features/src/main.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-fn main() {
- println!("Hello, world!");
-}