You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2023/10/03 06:15:42 UTC
[avro] branch avro-3653-arm64-on-github-actions updated (a36a83493 -> ae3b02272)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a change to branch avro-3653-arm64-on-github-actions
in repository https://gitbox.apache.org/repos/asf/avro.git
omit a36a83493 AVRO-3653: Use the new self-hosted ARM64 nodes provided by ASF Infra
new ae3b02272 AVRO-3653: Use the new self-hosted ARM64 nodes provided by ASF Infra
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a36a83493)
\
N -- N -- N refs/heads/avro-3653-arm64-on-github-actions (ae3b02272)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lang/csharp/common.props | 2 +-
lang/py/avro/test/test_bench.py | 4 ++--
lang/py/tox.ini | 5 ++---
3 files changed, 5 insertions(+), 6 deletions(-)
[avro] 01/01: AVRO-3653: Use the new self-hosted ARM64 nodes provided by ASF Infra
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch avro-3653-arm64-on-github-actions
in repository https://gitbox.apache.org/repos/asf/avro.git
commit ae3b02272ed0ebb3ac10606965d4271bce878ca6
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Mon Oct 2 16:37:26 2023 +0300
AVRO-3653: Use the new self-hosted ARM64 nodes provided by ASF Infra
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
.github/workflows/test-arm64.yml | 370 ++++++++++++++++-----------------------
lang/csharp/common.props | 2 +-
lang/py/avro/test/test_bench.py | 4 +-
lang/py/tox.ini | 5 +-
4 files changed, 159 insertions(+), 222 deletions(-)
diff --git a/.github/workflows/test-arm64.yml b/.github/workflows/test-arm64.yml
index 85f048e74..397b80af1 100644
--- a/.github/workflows/test-arm64.yml
+++ b/.github/workflows/test-arm64.yml
@@ -17,7 +17,8 @@ name: CI on Linux ARM64
on:
push:
- branches: [ master ]
+ branches:
+ - master
pull_request:
branches:
- master
@@ -25,11 +26,17 @@ on:
jobs:
java:
name: Java on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Setup Temurin JDK
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: 11
- name: Cache Local Maven Repository
uses: actions/cache@v3
@@ -40,89 +47,64 @@ jobs:
${{ runner.os }}-maven-
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --volume "${HOME}/.m2:/root/.m2" --workdir /avro/lang/java
- install: |
- apt-get update -q
- apt-get install -q -y openjdk-8-jdk wget tar
- run: |
- set -x
- export MAVEN_VERSION="3.9.1"
- wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
- tar zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz
- export M2_HOME=$PWD/apache-maven-$MAVEN_VERSION
- export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64"
- export PATH="$M2_HOME/bin:$JAVA_HOME/bin:$PATH"
- java -version
- mvn -version
- MAVEN_OPTS="-Dsurefire.excludes=*TestCustomCodec*,*TestAllCodecs*,*TestNettyServer*" ./build.sh clean test
+ run: |
+ set -x
+ cd lang/java
+ export MAVEN_VERSION="3.9.4"
+ wget https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
+ tar zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz
+ export M2_HOME=$PWD/apache-maven-$MAVEN_VERSION
+ export PATH="$M2_HOME/bin:$PATH"
+ java -version
+ mvn -version
+ #MAVEN_OPTS="-Dsurefire.excludes=*TestCustomCodec*,*TestAllCodecs*,*TestNettyServer*" ./build.sh clean test
+ ./build.sh clean test
c:
name: C on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y cmake liblzma-dev libsnappy-dev libjansson-dev zlib1g-dev pkg-config
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --workdir /avro/lang/c
- install: |
- apt-get update -q
- apt-get install -q -y cmake liblzma-dev libsnappy-dev libjansson-dev zlib1g-dev pkg-config
- run: |
- set -x
- ./build.sh clean test
+ run: |
+ set -x
+ cd lang/c
+ ./build.sh clean test
cplusplus:
name: C++ on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y gcc g++ libboost-all-dev cmake
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --workdir /avro/lang/c++
- install: |
- apt-get update -q
- apt-get install -q -y gcc g++ libboost-all-dev cmake
- run: |
- set -x
- ./build.sh clean test
+ run: |
+ set -x
+ cd lang/c++
+ ./build.sh clean test
csharp:
name: C# on Linux ARM64
- runs-on: ubuntu-latest
- env:
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Cache Nuget
uses: actions/cache@v3
@@ -132,69 +114,56 @@ jobs:
restore-keys: |
${{ runner.os }}-nuget-
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y wget libzstd-dev libicu-dev
+ wget https://dot.net/v1/dotnet-install.sh
+ bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1
+ bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0
+ bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0
+ bash ./dotnet-install.sh --channel "7.0" --install-dir "$HOME/.dotnet" # 7.0
+
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --volume "$HOME/.nuget/packages:/root/.nuget/packages" --workdir /avro/lang/csharp
- install: |
- apt-get update -q
- apt-get install -q -y wget libzstd-dev libicu-dev
- wget https://dot.net/v1/dotnet-install.sh
- bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1
- bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0
- bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0
- bash ./dotnet-install.sh --channel "7.0" --install-dir "$HOME/.dotnet" # 7.0
- run: |
- set -x
- export PATH=$HOME/.dotnet:$PATH
- dotnet --list-sdks
- ./build.sh clean test
+ run: |
+ set -x
+ export PATH=$HOME/.dotnet:$PATH
+ dotnet --list-sdks
+ cd lang/csharp
+ ./build.sh clean test
python:
name: Python on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+ # BENCH_MAX_WRITE_SECONDS: 300
+ # BENCH_MAX_READ_SECONDS: 300
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y software-properties-common
+ sudo add-apt-repository -y ppa:deadsnakes/ppa
+ sudo apt-get update -q
+ sudo apt-get install -q -y python3.8 python3.9 python3.10 python3.11 python3-pip git libbz2-dev libjansson-dev liblzma-dev libsnappy-dev libzstd-dev
+ python3 -m pip install --upgrade pip setuptools tox
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- BENCH_MAX_WRITE_SECONDS: 300
- BENCH_MAX_READ_SECONDS: 300
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --workdir /avro/lang/py
- install: |
- apt-get update -q
- apt-get install -q -y software-properties-common
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get update -q
- apt-get install -q -y python3.8 python3.9 python3.10 python3.11 python3-pip git libbz2-dev libjansson-dev liblzma-dev libsnappy-dev libzstd-dev
- python3 -m pip install --upgrade pip setuptools tox
- run: |
- set -x
- git config --global --add safe.directory /avro
- ./build.sh clean test
+ run: |
+ set -x
+ cd lang/py
+ ./build.sh clean test
ruby:
name: Ruby on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Cache gems
uses: actions/cache@v3
@@ -204,31 +173,24 @@ jobs:
restore-keys: |
${{ runner.os }}-gems-
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y ruby-dev bundler libsnappy-dev libyaml-dev
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --workdir /avro/lang/ruby
- install: |
- apt-get update -q
- apt-get install -q -y ruby-dev bundler libsnappy-dev libyaml-dev
- run: |
+ run: |
set -x
+ cd lang/ruby
./build.sh clean test
rust:
name: Rust on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Cache Cargo
uses: actions/cache@v3
@@ -237,105 +199,85 @@ jobs:
# and thus do not depend on the OS, arch nor rust version.
path: ~/.cargo
key: cargo-cache1-
+
- name: Cache Rust dependencies
uses: actions/cache@v3
with:
# these represent compiled steps of both dependencies and avro
# and thus are specific for a particular OS, arch and rust version.
- path: ~/target
- key: ${{ runner.os }}-target-cache1-${{ matrix.rust }}-
+ path: lang/rust/target
+ key: ${{ runner.os }}-target-cache1-stable-
- - name: Build
- uses: uraimo/run-on-arch-action@v2
+ - name: Rust Toolchain
+ uses: dtolnay/rust-toolchain@nightly
with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --volume "$HOME/.cargo:/root/.cargo" --volume "$HOME/target:/root/avro/target" --workdir /avro/lang/rust
- install: |
- set -x
- apt-get update -q
- apt-get install -q -y cargo rustc
- run: |
- set -x
- ./build.sh test
+ toolchain: stable
+ components: rustfmt
+ targets: x86_64-unknown-linux-gnu
+
+ - name: Build
+ run: |
+ set -x
+ cd lang/rust
+ ./build.sh test
perl:
name: Perl on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y libjansson-dev libcompress-raw-zlib-perl libcpan-uploader-perl libencode-perl libio-string-perl libjson-xs-perl libmodule-install-perl libmodule-install-readmefrompod-perl libobject-tiny-perl libperl-critic-perl libsnappy-dev libtest-exception-perl libtest-pod-perl cpanminus make gcc wget
+ cpanm Error::Simple
+ cpanm Regexp::Common
+ cpanm Try::Tiny
+ cpanm Compress::Zstd
+ cpanm Module::Install::Repository
+ cpanm inc::Module::Install
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --workdir /avro/lang/perl
- install: |
- apt-get update -q
- apt-get install -q -y libjansson-dev libcompress-raw-zlib-perl libcpan-uploader-perl libencode-perl libio-string-perl libjson-xs-perl libmodule-install-perl libmodule-install-readmefrompod-perl libobject-tiny-perl libperl-critic-perl libsnappy-dev libtest-exception-perl libtest-pod-perl cpanminus make gcc wget
- cpanm Error::Simple
- cpanm Regexp::Common
- cpanm Try::Tiny
- cpanm Compress::Zstd
- cpanm Module::Install::Repository
- cpanm inc::Module::Install
- run: |
- set -x
- ./build.sh clean test
+ run: |
+ set -x
+ cd lang/perl
+ ./build.sh clean test
php:
name: PHP on Linux ARM64
- runs-on: ubuntu-latest
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y wget php php-xml php-mbstring php-curl php-gmp php-bz2 unzip libtidy-dev libpq5
+ php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+ php -r "if (hash_file('sha384', 'composer-setup.php') === file_get_contents('https://composer.github.io/installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
+ php composer-setup.php --version=2.2.5
+ php -r "unlink('composer-setup.php');"
+ sudo mv composer.phar /usr/local/bin/composer
- name: Build
- uses: uraimo/run-on-arch-action@v2
- with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --workdir /avro/lang/php
- install: |
- apt-get update -q
- apt-get install -q -y wget php php-xml php-mbstring php-curl php-gmp php-bz2 unzip libtidy-dev libpq5
- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
- php -r "if (hash_file('sha384', 'composer-setup.php') === file_get_contents('https://composer.github.io/installer.sig')) { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
- php composer-setup.php --version=2.2.5
- php -r "unlink('composer-setup.php');"
- mv composer.phar /usr/local/bin/composer
- run: |
- set -x
- echo $PATH
- which composer
- composer --version
- ./build.sh clean test
+ run: |
+ set -x
+ cd lang/php
+ composer --version
+ ./build.sh clean test
javascript:
name: JavaScript on Linux ARM64
- runs-on: ubuntu-latest
- env:
- NODE_VERSION: "16.18.0"
+ runs-on: ["self-hosted", "asf-arm"]
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Cache Npm
uses: actions/cache@v3
@@ -345,23 +287,19 @@ jobs:
restore-keys: |
${{ runner.os }}-node-
- - name: Build
- uses: uraimo/run-on-arch-action@v2
+ - name: Setup Node
+ uses: actions/setup-node@v3
with:
- env: |
- GITHUB_WORKFLOW: ${{ github.workflow }}-${{ github.job }}
- arch: aarch64
- distro: ubuntu22.04
- githubToken: ${{ github.token }}
- dockerRunArgs: |
- --volume "${PWD}:/avro" --volume "$HOME/.npm:/root/.npm" --workdir /avro/lang/js
- install: |
- set -x
- apt-get update -q
- apt-get install -q -y wget tar xz-utils
- wget https://nodejs.org/dist/v${{ env.NODE_VERSION }}/node-v${{ env.NODE_VERSION }}-linux-arm64.tar.xz
- tar xf node-v${{ env.NODE_VERSION }}-linux-arm64.tar.xz
- run: |
- set -x
- export PATH=/node-v${{ env.NODE_VERSION }}-linux-arm64/bin:$PATH
- ./build.sh clean test
+ node-version: 18
+
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -q
+ sudo apt-get install -q -y wget tar xz-utils
+
+ - name: Build
+ run: |
+ set -x
+ cd lang/js
+ #export PATH=/node-v${{ env.NODE_VERSION }}-linux-arm64/bin:$PATH
+ ./build.sh clean test
diff --git a/lang/csharp/common.props b/lang/csharp/common.props
index 6dd567c3c..1874ea2a1 100644
--- a/lang/csharp/common.props
+++ b/lang/csharp/common.props
@@ -19,7 +19,7 @@
<PropertyGroup Label="Common Versions">
<!-- Read version from shared VERSION.txt file -->
- <AvroVersionFilePath>$(MSBuildThisFileDirectory)../../share/VERSION.txt</AvroVersionFilePath>
+ <AvroVersionFilePath>$(MSBuildThisFileDirectory)/../../share/VERSION.txt</AvroVersionFilePath>
<AvroVersion>$([System.IO.File]::ReadAllText($(AvroVersionFilePath)).Trim())</AvroVersion>
<!-- Parse out parts of the version -->
diff --git a/lang/py/avro/test/test_bench.py b/lang/py/avro/test/test_bench.py
index 05a4e64b8..460c5877e 100644
--- a/lang/py/avro/test/test_bench.py
+++ b/lang/py/avro/test/test_bench.py
@@ -55,8 +55,8 @@ SCHEMA = cast(
READER = avro.io.DatumReader(SCHEMA)
WRITER = avro.io.DatumWriter(SCHEMA)
NUMBER_OF_TESTS = 10000
-MAX_WRITE_SECONDS = 10 if platform.python_implementation() == "PyPy" else int(os.environ.get("BENCH_MAX_WRITE_SECONDS", 3))
-MAX_READ_SECONDS = 10 if platform.python_implementation() == "PyPy" else int(os.environ.get("BENCH_MAX_READ_SECONDS", 3))
+MAX_WRITE_SECONDS = 10 if platform.python_implementation() == "PyPy" else 3
+MAX_READ_SECONDS = 10 if platform.python_implementation() == "PyPy" else 3
class TestBench(unittest.TestCase):
diff --git a/lang/py/tox.ini b/lang/py/tox.ini
index a7d02f857..67395abbf 100644
--- a/lang/py/tox.ini
+++ b/lang/py/tox.ini
@@ -25,6 +25,7 @@ envlist =
py38
py39
py310
+ py311
pypy3.7
pypy3.8
pypy3.9
@@ -58,9 +59,7 @@ commands =
commands_post =
coverage combine --append
coverage report
-passenv =
- BENCH_MAX_READ_SECONDS
- BENCH_MAX_WRITE_SECONDS
+
[testenv:build]
##
# A virtual environment only used to build the wheel