You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2022/07/28 12:41:35 UTC
[arrow-rs] branch master updated: Break parquet CI into its own workflow, standardize step names (#2190)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 9d8f70071 Break parquet CI into its own workflow, standardize step names (#2190)
9d8f70071 is described below
commit 9d8f7007166dc5841d9eda640cbbecc2c43ad41f
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Thu Jul 28 08:41:30 2022 -0400
Break parquet CI into its own workflow, standardize step names (#2190)
* Break out parquet CI into its own workflow
* Fix up clippy
* Complete clippy move
* Remove linux test
* Fix arrow clippy features
* Update .github/workflows/parquet.yml
---
.github/workflows/arrow.yml | 27 ++++++---
.github/workflows/arrow_flight.yml | 22 ++++++-
.github/workflows/parquet.yml | 112 +++++++++++++++++++++++++++++++++++
.github/workflows/parquet_derive.yml | 20 ++++++-
.github/workflows/rust.yml | 72 ----------------------
parquet/src/encodings/rle.rs | 4 ++
6 files changed, 174 insertions(+), 83 deletions(-)
diff --git a/.github/workflows/arrow.yml b/.github/workflows/arrow.yml
index faef4ec11..bdca76f18 100644
--- a/.github/workflows/arrow.yml
+++ b/.github/workflows/arrow.yml
@@ -45,10 +45,10 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: stable
- - name: Test default features
+ - name: Test
run: |
cargo test -p arrow
- - name: Test all supported features
+ - name: Test --features=force_validate,prettyprint
run: |
cargo test -p arrow --features=force_validate,prettyprint
- name: Run examples
@@ -61,7 +61,7 @@ jobs:
# test compilaton features
linux-features:
- name: Feature Compatibility
+ name: Check Compilation
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -77,14 +77,23 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: stable
- - name: Test compilation with different features
+ - name: Check compilation
+ run: |
+ cargo check -p arrow
+ - name: Check compilation --no-default-features
run: |
cargo check -p arrow --no-default-features
+ - name: Check compilation --all-targets
+ run: |
cargo check -p arrow --all-targets
+ - name: Check compilation --no-default-features --all-targets
+ run: |
cargo check -p arrow --no-default-features --all-targets
+ - name: Check compilation --no-default-features --all-targets --features test_utils
+ run: |
cargo check -p arrow --no-default-features --all-targets --features test_utils
- # test the --features "simd" of the arrow crate. This requires nightly.
+ # test the --features "simd" of the arrow crate. This requires nightly Rust.
linux-test-simd:
name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
@@ -102,12 +111,14 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: nightly
- - name: Run tests
+ - name: Run tests --features "simd"
run: |
cargo test -p arrow --features "simd"
- - name: Check compilation with simd features
+ - name: Check compilation --features "simd"
run: |
cargo check -p arrow --features simd
+ - name: Check compilation --features simd --all-targets
+ run: |
cargo check -p arrow --features simd --all-targets
@@ -158,4 +169,4 @@ jobs:
rustup component add clippy
- name: Run clippy
run: |
- cargo clippy -p arrow --features test_common --features prettyprint --features=async --all-targets --workspace -- -D warnings
+ cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils --all-targets -- -D warnings
diff --git a/.github/workflows/arrow_flight.yml b/.github/workflows/arrow_flight.yml
index c101afa28..e3ca177cb 100644
--- a/.github/workflows/arrow_flight.yml
+++ b/.github/workflows/arrow_flight.yml
@@ -41,9 +41,27 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: stable
- - name: Tests with default features
+ - name: Test
run: |
cargo test -p arrow-flight
- - name: Tests with all features
+ - name: Test --all-features
run: |
cargo test -p arrow-flight --all-features
+
+ clippy:
+ name: Clippy
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Setup Clippy
+ run: |
+ rustup component add clippy
+ - name: Run clippy
+ run: |
+ cargo clippy -p arrow-flight --all-features -- -D warnings
diff --git a/.github/workflows/parquet.yml b/.github/workflows/parquet.yml
new file mode 100644
index 000000000..dfe58a666
--- /dev/null
+++ b/.github/workflows/parquet.yml
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+# tests for parquet crate
+name: "parquet"
+
+on:
+ pull_request:
+
+jobs:
+ # test the crate
+ linux-test:
+ name: Test
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ env:
+ # Disable full debug symbol generation to speed up CI build and keep memory down
+ # "1" means line tables only, which is useful for panic tracebacks.
+ RUSTFLAGS: "-C debuginfo=1"
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ submodules: true
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Test
+ run: |
+ cargo test -p parquet
+ - name: Test --all-features
+ run: |
+ cargo test -p parquet --all-features
+
+
+ # test compilaton
+ linux-features:
+ name: Check Compilation
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ env:
+ # Disable full debug symbol generation to speed up CI build and keep memory down
+ # "1" means line tables only, which is useful for panic tracebacks.
+ RUSTFLAGS: "-C debuginfo=1"
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ submodules: true
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Check compilation
+ run: |
+ cargo check -p parquet
+ - name: Check compilation --no-default-features
+ run: |
+ cargo check -p parquet --no-default-features
+ - name: Check compilation --no-default-features --features arrow
+ run: |
+ cargo check -p parquet --no-default-features --features arrow
+ - name: Check compilation --no-default-features --all-features
+ run: |
+ cargo check -p parquet --all-features
+ - name: Check compilation --all-targets
+ run: |
+ cargo check -p parquet --all-targets
+ - name: Check compilation --no-default-features --all-targets
+ run: |
+ cargo check -p parquet --no-default-features --all-targets
+ - name: Check compilation --no-default-features --features-arrow --all-targets
+ run: |
+ cargo check -p parquet --no-default-features --features arrow --all-targets
+
+ clippy:
+ name: Clippy
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Setup Clippy
+ run: |
+ rustup component add clippy
+ - name: Run clippy
+ run: |
+ # Only run clippy for the library at this time,
+ # as there are clippy errors for other targets
+ cargo clippy -p parquet --all-features --lib -- -D warnings
+ # https://github.com/apache/arrow-rs/issues/1254
+ #cargo clippy -p parquet --all-targets --all-features -- -D warnings
diff --git a/.github/workflows/parquet_derive.yml b/.github/workflows/parquet_derive.yml
index 6ba3138e2..f8775e24e 100644
--- a/.github/workflows/parquet_derive.yml
+++ b/.github/workflows/parquet_derive.yml
@@ -41,6 +41,24 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: stable
- - name: Test crate
+ - name: Test
run: |
cargo test -p parquet_derive
+
+ clippy:
+ name: Clippy
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Setup Clippy
+ run: |
+ rustup component add clippy
+ - name: Run clippy
+ run: |
+ cargo clippy -p parquet_derive --all-features -- -D warnings
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 63b1f3d4d..bd31982ef 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -26,50 +26,6 @@ on:
jobs:
- # test the crate
- linux-test:
- name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [ amd64 ]
- rust: [ stable ]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
- PARQUET_TEST_DATA: /__w/arrow-rs/arrow-rs/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{ matrix.rust }}
- - name: Run tests
- run: |
- # run tests on all workspace members with default feature list
- cargo test
- - name: Re-run tests on parquet crate with all features
- run: |
- cargo test -p parquet --all-features
- - name: Test compilation of parquet library crate with different feature combinations
- run: |
- cargo check -p parquet
- cargo check -p parquet --no-default-features
- cargo check -p parquet --no-default-features --features arrow
- cargo check -p parquet --all-features
- - name: Test compilation of parquet targets with different feature combinations
- run: |
- cargo check -p parquet --all-targets
- cargo check -p parquet --no-default-features --all-targets
- cargo check -p parquet --no-default-features --features arrow --all-targets
-
-
windows-and-macos:
name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
runs-on: ${{ matrix.os }}
@@ -94,34 +50,6 @@ jobs:
export RUSTFLAGS="-C debuginfo=0"
cargo test
- clippy:
- name: Clippy
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [ amd64 ]
- rust: [ stable ]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{ matrix.rust }}
- - name: Setup Clippy
- run: |
- rustup component add clippy
- - name: Run clippy
- run: |
- cargo clippy --features test_common --features prettyprint --features=async --all-targets --workspace -- -D warnings
-
check_benches:
name: Check Benchmarks (but don't run them)
runs-on: ubuntu-latest
diff --git a/parquet/src/encodings/rle.rs b/parquet/src/encodings/rle.rs
index 808c8f0d4..8a19dd545 100644
--- a/parquet/src/encodings/rle.rs
+++ b/parquet/src/encodings/rle.rs
@@ -182,6 +182,10 @@ impl RleEncoder {
self.bit_writer.bytes_written()
}
+ pub fn is_empty(&self) -> bool {
+ self.bit_writer.bytes_written() == 0
+ }
+
#[inline]
pub fn consume(mut self) -> Result<Vec<u8>> {
self.flush()?;