You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by er...@apache.org on 2020/11/24 19:40:35 UTC
[incubator-superset] branch master updated: build: split python CI
jobs (#11794)
This is an automated email from the ASF dual-hosted git repository.
erikrit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 67c4447 build: split python CI jobs (#11794)
67c4447 is described below
commit 67c44479b56ad496417eeb7901b4b1a470ef4d55
Author: Erik Ritter <er...@airbnb.com>
AuthorDate: Tue Nov 24 11:40:16 2020 -0800
build: split python CI jobs (#11794)
* build: split python CI jobs
* Rename superset-python.yml to superset-python-misc.yml
---
.github/workflows/superset-python-misc.yml | 72 +++++
.github/workflows/superset-python-mysql.yml | 51 ++++
.github/workflows/superset-python-postgres.yml | 52 ++++
.github/workflows/superset-python-presto-hive.yml | 119 ++++++++
.github/workflows/superset-python-sqlite.yml | 44 +++
.github/workflows/superset-python.yml | 322 ----------------------
6 files changed, 338 insertions(+), 322 deletions(-)
diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml
new file mode 100644
index 0000000..8db08ed
--- /dev/null
+++ b/.github/workflows/superset-python-misc.yml
@@ -0,0 +1,72 @@
+# Python Misc unit tests
+name: Python Misc
+
+on: [push, pull_request]
+
+jobs:
+ lint:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ - name: pylint
+ # `-j 0` run Pylint in parallel
+ run: pylint -j 0 superset
+
+ pre-commit:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/integration.txt
+ - name: pre-commit
+ run: pre-commit run --all-files
+
+ babel-extract:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/base.txt
+ - name: Test babel extraction
+ run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
diff --git a/.github/workflows/superset-python-mysql.yml b/.github/workflows/superset-python-mysql.yml
new file mode 100644
index 0000000..a8b2394
--- /dev/null
+++ b/.github/workflows/superset-python-mysql.yml
@@ -0,0 +1,51 @@
+# Python MySQL unit tests
+name: Python MySQL
+
+on: [push, pull_request]
+
+jobs:
+ test-mysql:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI: |
+ mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
+ services:
+ mysql:
+ image: mysql:5.7
+ env:
+ MYSQL_ROOT_PASSWORD: root
+ ports:
+ - 13306:3306
+ redis:
+ image: redis:5-alpine
+ options: --entrypoint redis-server
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ setup-mysql
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (MySQL)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
diff --git a/.github/workflows/superset-python-postgres.yml b/.github/workflows/superset-python-postgres.yml
new file mode 100644
index 0000000..95bd413
--- /dev/null
+++ b/.github/workflows/superset-python-postgres.yml
@@ -0,0 +1,52 @@
+# Python Postgres unit tests
+name: Python Postgres
+
+on: [push, pull_request]
+
+jobs:
+ test-postgres:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7, 3.8]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+ services:
+ postgres:
+ image: postgres:10-alpine
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15432:5432
+ redis:
+ image: redis:5-alpine
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ setup-postgres
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (PostgreSQL)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml
new file mode 100644
index 0000000..e723c6c
--- /dev/null
+++ b/.github/workflows/superset-python-presto-hive.yml
@@ -0,0 +1,119 @@
+# Python Presto/Hive unit tests
+name: Python Presto/Hive
+
+on: [push, pull_request]
+
+jobs:
+ test-postgres-presto:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ # run unit tests in multiple version just for fun
+ python-version: [3.8]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+ SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
+ services:
+ postgres:
+ image: postgres:10-alpine
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15432:5432
+ presto:
+ image: prestosql/presto:339
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15433:8080
+ redis:
+ image: redis:5-alpine
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ setup-postgres
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (PostgreSQL)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
+
+ test-postgres-hive:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ # run unit tests in multiple version just for fun
+ python-version: [3.7, 3.8]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+ SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
+ UPLOAD_FOLDER: /tmp/.superset/uploads/
+ services:
+ postgres:
+ image: postgres:10-alpine
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15432:5432
+ redis:
+ image: redis:5-alpine
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Create csv upload directory
+ run: sudo mkdir -p /tmp/.superset/uploads
+ - name: Give write access to the csv upload directory
+ run: sudo chown -R $USER:$USER /tmp/.superset
+ - name: Start hadoop and hive
+ run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ setup-postgres
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (PostgreSQL)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
diff --git a/.github/workflows/superset-python-sqlite.yml b/.github/workflows/superset-python-sqlite.yml
new file mode 100644
index 0000000..63cfed3
--- /dev/null
+++ b/.github/workflows/superset-python-sqlite.yml
@@ -0,0 +1,44 @@
+# Python Sqlite unit tests
+name: Python Sqlite
+
+on: [push, pull_request]
+
+jobs:
+ test-sqlite:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI: |
+ sqlite:///${{ github.workspace }}/.temp/unittest.db
+ services:
+ redis:
+ image: redis:5-alpine
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ mkdir ${{ github.workspace }}/.temp
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (SQLite)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
diff --git a/.github/workflows/superset-python.yml b/.github/workflows/superset-python.yml
deleted file mode 100644
index 4221c88..0000000
--- a/.github/workflows/superset-python.yml
+++ /dev/null
@@ -1,322 +0,0 @@
-# Python unit tests
-name: Python
-
-on: [push, pull_request]
-
-jobs:
- lint:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- - name: pylint
- # `-j 0` run Pylint in parallel
- run: pylint -j 0 superset
-
- pre-commit:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/integration.txt
- - name: pre-commit
- run: pre-commit run --all-files
-
- babel-extract:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/base.txt
- - name: Test babel extraction
- run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
-
- test-postgres-presto:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- # run unit tests in multiple version just for fun
- python-version: [3.8]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI:
- postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
- SUPERSET__SQLALCHEMY_EXAMPLES_URI:
- presto://localhost:15433/memory/default
- services:
- postgres:
- image: postgres:10-alpine
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15432:5432
- presto:
- image: prestosql/presto:339
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15433:8080
- redis:
- image: redis:5-alpine
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- setup-postgres
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (PostgreSQL)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python
-
- test-postgres-hive:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- # run unit tests in multiple version just for fun
- python-version: [3.7, 3.8]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI:
- postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
- SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
- UPLOAD_FOLDER: /tmp/.superset/uploads/
- services:
- postgres:
- image: postgres:10-alpine
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15432:5432
- redis:
- image: redis:5-alpine
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Create csv upload directory
- run: sudo mkdir -p /tmp/.superset/uploads
- - name: Give write access to the csv upload directory
- run: sudo chown -R $USER:$USER /tmp/.superset
- - name: Start hadoop and hive
- run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- setup-postgres
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (PostgreSQL)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python
-
- test-postgres:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7, 3.8]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI:
- postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
- services:
- postgres:
- image: postgres:10-alpine
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15432:5432
- redis:
- image: redis:5-alpine
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- setup-postgres
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (PostgreSQL)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python
-
- test-mysql:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI: |
- mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
- services:
- mysql:
- image: mysql:5.7
- env:
- MYSQL_ROOT_PASSWORD: root
- ports:
- - 13306:3306
- redis:
- image: redis:5-alpine
- options: --entrypoint redis-server
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- setup-mysql
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (MySQL)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python
-
- test-sqlite:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI: |
- sqlite:///${{ github.workspace }}/.temp/unittest.db
- services:
- redis:
- image: redis:5-alpine
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- mkdir ${{ github.workspace }}/.temp
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (SQLite)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python