You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2021/04/18 18:36:08 UTC
[arrow-datafusion] 01/01: Fixed CI.
This is an automated email from the ASF dual-hosted git repository.
jorgecarleitao pushed a commit to branch fix-ci
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit 21170cb87a9524ec3f828d989452218ff7195596
Author: Jorge C. Leitao <jo...@gmail.com>
AuthorDate: Sun Apr 18 18:30:39 2021 +0000
Fixed CI.
---
.github/workflows/cancel.yml | 77 --------------
.github/workflows/dev.yml | 59 +----------
.github/workflows/integration.yml | 83 ----------------
.github/workflows/rust.yml | 204 +++-----------------------------------
4 files changed, 18 insertions(+), 405 deletions(-)
diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
index de980eb..e1c6ed9 100644
--- a/.github/workflows/cancel.yml
+++ b/.github/workflows/cancel.yml
@@ -31,20 +31,6 @@ jobs:
# Unfortunately, we need to define a separate cancellation step for
# each workflow where we want to cancel stale runs.
- uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale C++ runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: cpp.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale C# runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: csharp.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
name: "Cancel stale Dev runs"
with:
cancelMode: allDuplicates
@@ -52,69 +38,6 @@ jobs:
workflowFileName: dev.yml
skipEventTypes: '["push", "schedule"]'
- uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Go runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: go.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Integration runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: integration.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Java JNI runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: java_jni.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Java runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: java.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale JS runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: js.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Julia runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: julia.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Python runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: python.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale R runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: r.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Ruby runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: ruby.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
name: "Cancel stale Rust runs"
with:
cancelMode: allDuplicates
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index d1b0184..9d8146a 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -31,66 +31,13 @@ jobs:
lint:
name: Lint C++, Python, R, Rust, Docker, RAT
runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
+ - uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Setup Archery
run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ubuntu-lint
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
- continue-on-error: true
- run: archery docker push ubuntu-lint
-
- release:
- name: Source Release and Merge Script
- runs-on: ubuntu-20.04
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- env:
- GIT_AUTHOR_NAME: Github Actions
- GIT_AUTHOR_EMAIL: github@actions
- GIT_COMMITTER_NAME: Github Actions
- GIT_COMMITTER_EMAIL: github@actions
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Python
- uses: actions/setup-python@v1
- with:
- python-version: '3.6'
- - name: Install Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: '2.6'
- - name: Install Dependencies
- shell: bash
- run: |
- pip install cython setuptools pytest jira
- - name: Run Release Test
- shell: bash
- run: |
- ci/scripts/release_test.sh $(pwd)
- - name: Run Merge Script Test
- shell: bash
- run: |
- pytest -v dev/test_merge_arrow_pr.py
+ - name: Lint
+ run: archery lint --rat
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
deleted file mode 100644
index 2011255..0000000
--- a/.github/workflows/integration.yml
+++ /dev/null
@@ -1,83 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Integration
-
-on:
- push:
- paths:
- - '.github/workflows/integration.yml'
- - 'ci/**'
- - 'dev/archery/**'
- - 'go/**'
- - 'integration/**'
- - 'js/**'
- - 'cpp/**'
- - 'java/**'
- - 'format/**'
- - 'rust/**'
- pull_request:
- paths:
- - '.github/workflows/integration.yml'
- - 'ci/**'
- - 'dev/archery/**'
- - 'go/**'
- - 'integration/**'
- - 'js/**'
- - 'cpp/**'
- - 'java/**'
- - 'format/**'
- - 'rust/**'
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: AMD64 Conda Integration Test
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: conda-${{ hashFiles('cpp/**') }}
- restore-keys: conda-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: archery docker run conda-integration
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
- continue-on-error: true
- run: archery docker push conda-integration
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4bb17a2..3f1031c 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -123,7 +123,6 @@ jobs:
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust
# run tests on all workspace members with default feature list
cargo test
# test datafusion examples
@@ -131,14 +130,6 @@ jobs:
cargo test --no-default-features
cargo run --example csv_sql
cargo run --example parquet_sql
- cd ..
- cd arrow
- # re-run tests on arrow workspace with additional features
- cargo test --features=prettyprint
- cargo run --example builders
- cargo run --example dynamic_types
- cargo run --example read_csv
- cargo run --example read_csv_infer_schema
# Ballista is currently not part of the main workspace so requires a separate test step
- name: Run Ballista tests
run: |
@@ -148,49 +139,6 @@ jobs:
# snmalloc requires cmake so build without default features
cargo test --no-default-features
- # test the --features "simd" of the arrow crate. This requires nightly.
- linux-test-simd:
- name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [nightly-2021-03-24]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Run tests
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd rust/arrow
- cargo test --features "simd"
-
windows-and-macos:
name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
runs-on: ${{ matrix.os }}
@@ -216,9 +164,23 @@ jobs:
export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data
# do not produce debug symbols to keep memory usage down
export RUSTFLAGS="-C debuginfo=0"
- cd rust
cargo test
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup toolchain
+ run: |
+ rustup toolchain install stable
+ rustup default stable
+ rustup component add rustfmt
+ - name: Run
+ run: cargo fmt --all -- --check
+
clippy:
name: Clippy
needs: [linux-build-lib]
@@ -258,7 +220,6 @@ jobs:
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust
cargo clippy --all-targets --workspace -- -D warnings -A clippy::redundant_field_names
miri-checks:
@@ -290,7 +251,6 @@ jobs:
RUST_LOG: 'trace'
run: |
export MIRIFLAGS="-Zmiri-disable-isolation"
- cd rust
cargo miri setup
cargo clean
# Ignore MIRI errors until we can get a clean run
@@ -330,141 +290,7 @@ jobs:
# 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
# see https://github.com/xd009642/tarpaulin/issues/618
cargo install --version 0.16.0 cargo-tarpaulin
- cd rust
cargo tarpaulin --out Xml
- name: Report coverage
continue-on-error: true
run: bash <(curl -s https://codecov.io/bash)
-
- # test FFI against the C-Data interface exposed by pyarrow
- pyarrow-integration-test:
- name: Test Pyarrow C Data Interface
- runs-on: ubuntu-latest
- strategy:
- matrix:
- rust: [stable]
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt clippy
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /home/runner/.cargo
- key: cargo-maturin-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /home/runner/target
- # this key is not equal because maturin uses different compilation flags.
- key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}-
- - uses: actions/setup-python@v2
- with:
- python-version: '3.7'
- - name: Install Python dependencies
- run: python -m pip install --upgrade pip setuptools wheel
- - name: Run tests
- run: |
- export CARGO_HOME="/home/runner/.cargo"
- export CARGO_TARGET_DIR="/home/runner/target"
-
- cd rust/arrow-pyarrow-integration-testing
-
- python -m venv venv
- source venv/bin/activate
-
- pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0
- maturin develop
- python -m unittest discover tests
-
- # test the arrow crate builds against wasm32 in stable rust
- wasm32-build:
- name: Build wasm32 on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [nightly-2021-03-24]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup override set ${{ matrix.rust }}
- rustup component add rustfmt
- rustup target add wasm32-unknown-unknown
- - name: Build arrow crate
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd rust/arrow
- cargo build --target wasm32-unknown-unknown
-
- # test the projects can build without default features
- default-build:
- name: Check No Defaults on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup override set ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Build arrow crate
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd rust/arrow
- cargo check --all-targets --no-default-features