You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by so...@apache.org on 2022/03/12 13:27:54 UTC
[dolphinscheduler] branch dev updated: [python] Add tox for out of box test (#8837)
This is an automated email from the ASF dual-hosted git repository.
songjian pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 90f9aed [python] Add tox for out of box test (#8837)
90f9aed is described below
commit 90f9aed11dafe1617b5b80418573b645f3c79072
Author: Jiajie Zhong <zh...@hotmail.com>
AuthorDate: Sat Mar 12 21:27:36 2022 +0800
[python] Add tox for out of box test (#8837)
* [python] Add tox for out of box test
* Add `tox` for easy and complete test
* Correct package data attribute
* Add more dirs in flake8 ignore
* Change GA from raw to tox
* Fix env setter
* Temp skip sanity
* Rm not exists job
* Add code test
* fix
* rm all need
* fix
* fix
* fix
* fix
* fix
* fix
* Migrate pip upgrede to tox setting
* Change install_commands to commands
* fix config of install command
* Add GA env to do that
* Fix env error
* Finial commit
* remove duplicate
* Change doc
* Change dependence of ci
* Change dependence of ci
---
.github/workflows/py-ci.yml | 79 +++++++++++-----------
.gitignore | 1 +
.../pydolphinscheduler/.coveragerc | 4 +-
dolphinscheduler-python/pydolphinscheduler/.flake8 | 4 +-
.../pydolphinscheduler/DEVELOP.md | 73 ++++++++++++++++----
.../pydolphinscheduler/setup.py | 3 +-
.../pydolphinscheduler/{.coveragerc => tox.ini} | 44 ++++++++----
7 files changed, 135 insertions(+), 73 deletions(-)
diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml
index db430ca..d10d255 100644
--- a/.github/workflows/py-ci.yml
+++ b/.github/workflows/py-ci.yml
@@ -25,22 +25,21 @@ on:
paths:
- 'dolphinscheduler-python/**'
+concurrency:
+ group: py-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
defaults:
run:
working-directory: dolphinscheduler-python/pydolphinscheduler
+# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
+env:
+ DEPENDENCES: pip setuptools wheel tox
+
jobs:
- sanity:
- name: Sanity
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Sanity Check
- uses: ./.github/actions/sanity-check
lint:
- name: Code Style
+ name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
@@ -48,19 +47,15 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: 3.7
- - name: Install Development Dependences
- run: pip install -e .[style]
- - name: Run Isort Checking
- run: isort --check .
- - name: Run Black Checking
- run: black --check .
- - name: Run Flake8 Checking
- run: flake8
+ - name: Install Dependences
+ run: |
+ python -m pip install --upgrade ${{ env.DEPENDENCES }}
+ - name: Run All Lint Check
+ run: |
+ python -m tox -vv -e lint
pytest:
name: Pytest
- needs:
- - lint
- - sanity
+ needs: lint
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -73,16 +68,15 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- - name: Install dependencies & pydolphinscheduler
+ - name: Install Dependences
run: |
- pip install -e .[test]
- - name: Run tests
+ python -m pip install --upgrade ${{ env.DEPENDENCES }}
+ - name: Run All Tests
run: |
- pytest
- coverage:
- name: Tests coverage
- needs:
- - pytest
+ python -m tox -vv -e code-test
+ doc-build:
+ name: Docs Build Test
+ needs: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
@@ -90,14 +84,17 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: 3.7
- - name: Install Development Dependences
+ - name: Install Dependences
run: |
- pip install -e .[test]
- - name: Run Tests && Check coverage
- run: coverage run && coverage report
- doc-build:
- name: Document Build Test
- needs: pytest
+ python -m pip install --upgrade ${{ env.DEPENDENCES }}
+ - name: Run Tests Build Docs
+ run: |
+ python -m tox -vv -e doc-build-test
+ verify-local-ci:
+ name: Local CI
+ needs:
+ - pytest
+ - doc-build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
@@ -105,9 +102,9 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: 3.7
- - name: Install Development Dependences
+ - name: Install Dependences
+ run: |
+ python -m pip install --upgrade ${{ env.DEPENDENCES }}
+ - name: Run Tests Build Docs
run: |
- pip install -e .[doc]
- - name: Test Build Document
- working-directory: dolphinscheduler-python/pydolphinscheduler/docs
- run: make clean && make html
+ python -m tox -vv -e local-ci
diff --git a/.gitignore b/.gitignore
index 0a0aaaa..765259a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -55,6 +55,7 @@ dolphinscheduler-worker/logs
# ------------------
# Cache
__pycache__/
+.tox/
# Build
build/
diff --git a/dolphinscheduler-python/pydolphinscheduler/.coveragerc b/dolphinscheduler-python/pydolphinscheduler/.coveragerc
index 52286d6..1620509 100644
--- a/dolphinscheduler-python/pydolphinscheduler/.coveragerc
+++ b/dolphinscheduler-python/pydolphinscheduler/.coveragerc
@@ -20,8 +20,10 @@ command_line = -m pytest
omit =
# Ignore all test cases in tests/
tests/*
+ # Ignore examples directory
+ */pydolphinscheduler/examples/*
# TODO. Temporary ignore java_gateway file, because we could not find good way to test it.
- src/pydolphinscheduler/java_gateway.py
+ */pydolphinscheduler/java_gateway.py
[report]
# Don’t report files that are 100% covered
diff --git a/dolphinscheduler-python/pydolphinscheduler/.flake8 b/dolphinscheduler-python/pydolphinscheduler/.flake8
index 7f659a2..9fbada8 100644
--- a/dolphinscheduler-python/pydolphinscheduler/.flake8
+++ b/dolphinscheduler-python/pydolphinscheduler/.flake8
@@ -26,7 +26,9 @@ exclude =
old,
build,
dist,
- htmlcov
+ htmlcov,
+ .tox,
+ dist,
ignore =
# It's clear and not need to add docstring
D107, # D107: Don't require docstrings on __init__
diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
index ef7dfa4..4158b5b 100644
--- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
+++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
@@ -60,7 +60,53 @@ pydolphinscheduler tasks object, we use tasks to define exact job we want Dolphi
we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler
and would be implemented in the further.
-## Code Style
+## Test Your Code
+
+Linting and tests is very important for open source project, so we pay more attention to it. We have continuous
+integration service run by GitHub Action to test whether the patch is good or not, which you could jump to
+section [With GitHub Action](#with-github-action) see more detail.
+
+And to make more convenience to local tests, we also have the way to run your [test automated with tox](#automated-testing-with-tox)
+locally. It is helpful when your try to find out the detail when continuous integration in GitHub Action failed,
+or you have a great patch and want to test local first.
+
+Besides [automated testing with tox](#automated-testing-with-tox) locally, we also have a [manual way](#manually)
+run tests. And it is scattered commands to reproduce each step of the integration test we told about.
+
+* Remote
+ * [With GitHub Action](#with-github-action)
+* Local
+ * [Automated Testing With tox](#automated-testing-with-tox)
+ * [Manually](#manually)
+
+### With GitHub Action
+
+GitHub Action test in various environment for pydolphinscheduler, including different python version in
+`3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. It will trigger and run automatically when you
+submit pull requests to `apache/dolphinscheduler`.
+
+### Automated Testing With tox
+
+[tox](https://tox.wiki) is a package aims to automate and standardize testing in Python, both our continuous
+integration and local test use it to run actual task. To use it, you should install it first
+
+```shell
+python -m pip install --upgrade tox
+```
+
+After installation, you could run a single command to run all the tests, it is almost like test in GitHub Action
+but not so much different environment.
+
+```shell
+tox -e local-ci
+```
+
+It will take a while when you run it the first time, because it has to install dependencies and make some prepare,
+and the next time you run it will be faster.
+
+### Manually
+
+#### Code Style
We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code
formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea],
@@ -74,33 +120,30 @@ GitHub, you could also run static check locally.
# but Flake8 just hint when code style not match pep8
# Run Isort
-isort .
+python -m isort .
# Run Black
-black .
+python -m black .
# Run Flake8
-flake8
+python -m flake8
```
-## Testing
+#### Testing
-pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create
-pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`.
-
-To test locally, you could directly run pytest after set `PYTHONPATH`
+pydolphinscheduler using [pytest][pytest] to run all tests in directory `tests`. You could run tests by the commands
```shell
-PYTHONPATH=src/ pytest
+python -m pytest --cov=pydolphinscheduler --cov-config=.coveragerc tests/
```
-We try to keep pydolphinscheduler usable through unit test coverage. 90% test coverage is our target, but for
-now, we require test coverage up to 85%, and each pull request leas than 85% would fail our CI step
-`Tests coverage`. We use [coverage][coverage] to check our test coverage, and you could check it locally by
-run command.
+Besides run tests, it will also check the unit test [coverage][coverage] threshold, for now when test cover less than 90%
+will fail the coverage, as well as our GitHub Action.
+
+The command above will check test coverage automatically, and you could also test the coverage by command.
```shell
-coverage run && coverage report
+python -m coverage run && python -m coverage report
```
It would not only run unit test but also show each file coverage which cover rate less than 100%, and `TOTAL`
diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py
index 7b5cda8..ffb0888 100644
--- a/dolphinscheduler-python/pydolphinscheduler/setup.py
+++ b/dolphinscheduler-python/pydolphinscheduler/setup.py
@@ -57,6 +57,7 @@ test = [
"pytest>=6.2",
"freezegun>=1.1",
"coverage>=6.1",
+ "pytest-cov>=3.0",
]
style = [
@@ -144,7 +145,7 @@ setup(
package_dir={"": "src"},
include_package_data=True,
package_data={
- "examples": ["examples.tutorial.py"],
+ "pydolphinscheduler": ["core/default_config.yaml"],
},
platforms=["any"],
classifiers=[
diff --git a/dolphinscheduler-python/pydolphinscheduler/.coveragerc b/dolphinscheduler-python/pydolphinscheduler/tox.ini
similarity index 54%
copy from dolphinscheduler-python/pydolphinscheduler/.coveragerc
copy to dolphinscheduler-python/pydolphinscheduler/tox.ini
index 52286d6..865c862 100644
--- a/dolphinscheduler-python/pydolphinscheduler/.coveragerc
+++ b/dolphinscheduler-python/pydolphinscheduler/tox.ini
@@ -15,18 +15,34 @@
# specific language governing permissions and limitations
# under the License.
-[run]
-command_line = -m pytest
-omit =
- # Ignore all test cases in tests/
- tests/*
- # TODO. Temporary ignore java_gateway file, because we could not find good way to test it.
- src/pydolphinscheduler/java_gateway.py
+[tox]
+envlist = local-ci, lint, doc-build-test, code-test, py{36,37,38,39}
-[report]
-# Don’t report files that are 100% covered
-skip_covered = True
-show_missing = True
-precision = 2
-# Report will fail when coverage under 90.00%
-fail_under = 90
+[testenv]
+whitelist_externals = make
+
+[testenv:lint]
+extras = style
+commands =
+ python -m isort --check .
+ python -m black --check .
+ python -m flake8
+
+[testenv:code-test]
+extras = test
+# Run both tests and coverage
+commands =
+ python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/
+
+[testenv:doc-build-test]
+extras = doc
+commands =
+ make -C {toxinidir}/docs clean
+ make -C {toxinidir}/docs html
+
+[testenv:local-ci]
+extras = dev
+commands =
+ {[testenv:lint]commands}
+ {[testenv:code-test]commands}
+ {[testenv:doc-build-test]commands}