You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/03/13 01:01:00 UTC
(superset) branch kill-pip-compile-multi updated (0944e7777b -> 698052cf10)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a change to branch kill-pip-compile-multi
in repository https://gitbox.apache.org/repos/asf/superset.git
from 0944e7777b entry points
new 012d2b5c51 drying actions
new 698052cf10 drying actions
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/actions/setup-python.yml | 36 ++
.../workflows/superset-python-integrationtest.yml | 28 +-
.github/workflows/superset-python-unittest.yml | 18 +-
pyproject.toml | 6 +
requirements/base.txt | 1 -
requirements/development.txt | 2 -
requirements/pip-compile-custom.sh | 32 --
requirements/pip-compile-superset.py | 148 ++++++++
requirements/pip-compile-superset.sh | 33 --
requirements/requirements-custom-example.in | 2 +-
...lopment.txt => requirements-custom-example.txt} | 386 +++++++--------------
11 files changed, 327 insertions(+), 365 deletions(-)
create mode 100644 .github/actions/setup-python.yml
delete mode 100644 requirements/pip-compile-custom.sh
create mode 100644 requirements/pip-compile-superset.py
delete mode 100755 requirements/pip-compile-superset.sh
copy requirements/{development.txt => requirements-custom-example.txt} (59%)
(superset) 02/02: drying actions
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch kill-pip-compile-multi
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 698052cf10d7ad65bad5b535feba5d4d5317a42e
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Tue Mar 12 18:00:36 2024 -0700
drying actions
---
.github/actions/setup-python.yml | 36 ++
requirements/requirements-custom-example.txt | 689 +++++++++++++++++++++++++++
2 files changed, 725 insertions(+)
diff --git a/.github/actions/setup-python.yml b/.github/actions/setup-python.yml
new file mode 100644
index 0000000000..cad0a7b5b2
--- /dev/null
+++ b/.github/actions/setup-python.yml
@@ -0,0 +1,36 @@
+name: 'Setup Python Environment'
+description: 'Set up Python and install dependencies with optional configurations.'
+inputs:
+ python-version:
+ description: 'Python version to set up.'
+ required: true
+ default: '3.9'
+ cache:
+ description: 'Cache dependencies. Options: pip'
+ required: false
+ default: 'pip'
+ requirements-type:
+ description: 'Type of requirements to install. Options: base, development, default'
+ required: false
+ default: 'dev'
+
+runs:
+ using: 'composite'
+ steps:
+ - name: Set up Python ${{ inputs.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ inputs.python-version }}
+ cache: ${{ inputs.cache }}
+ - name: Install dependencies
+ run: |
+ pip install --upgrade pip setuptools wheel
+ if [ "${{ inputs.requirements-type }}" = "dev" ]; then
+ pip install -r requirements/development.txt
+ elif [ "${{ inputs.requirements-type }}" = "base" ]; then
+ pip install -r requirements/base.txt
+ else
+ pip install -r requirements.txt
+ fi
+ pip install -e .
+ shell: bash
diff --git a/requirements/requirements-custom-example.txt b/requirements/requirements-custom-example.txt
new file mode 100644
index 0000000000..a592f930af
--- /dev/null
+++ b/requirements/requirements-custom-example.txt
@@ -0,0 +1,689 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+# pip-compile --output-file=requirements/requirements-custom-example.txt requirements/requirements-custom-example.in
+#
+-e file:///Users/max/code/superset
+ # via file:///users/max/code/superset
+alembic==1.6.5
+ # via flask-migrate
+amqp==5.1.1
+ # via kombu
+apispec[yaml]==6.3.0
+ # via
+ # apispec
+ # flask-appbuilder
+appnope==0.1.4
+ # via ipython
+apsw==3.42.0.1
+ # via shillelagh
+astroid==2.15.8
+ # via pylint
+asttokens==2.2.1
+ # via stack-data
+async-timeout==4.0.2
+ # via redis
+attrs==23.1.0
+ # via
+ # cattrs
+ # jsonlines
+ # jsonschema
+ # requests-cache
+babel==2.9.1
+ # via flask-babel
+backcall==0.2.0
+ # via ipython
+backoff==1.11.1
+ # via apache-superset
+bcrypt==4.0.1
+ # via paramiko
+billiard==4.2.0
+ # via celery
+boto3==1.29.7
+ # via
+ # -r requirements/requirements-custom-example.in
+ # tabulator
+botocore==1.32.7
+ # via
+ # boto3
+ # s3transfer
+bottleneck==1.3.7
+ # via pandas
+brotli==1.0.9
+ # via flask-compress
+cached-property==1.5.2
+ # via tableschema
+cachelib==0.9.0
+ # via
+ # flask-caching
+ # flask-session
+cachetools==5.3.2
+ # via
+ # google-auth
+ # tox
+cattrs==23.2.1
+ # via requests-cache
+celery==5.3.6
+ # via apache-superset
+certifi==2023.7.22
+ # via requests
+cffi==1.15.1
+ # via
+ # cryptography
+ # pynacl
+cfgv==3.3.1
+ # via pre-commit
+chardet==5.1.0
+ # via
+ # tabulator
+ # tox
+charset-normalizer==3.2.0
+ # via requests
+cherrytree==2.0.1
+ # via -r requirements/requirements-custom-example.in
+click==8.1.3
+ # via
+ # apache-superset
+ # celery
+ # cherrytree
+ # click-didyoumean
+ # click-option-group
+ # click-plugins
+ # click-repl
+ # flask
+ # flask-appbuilder
+ # tableschema
+ # tabulator
+click-didyoumean==0.3.0
+ # via celery
+click-option-group==0.5.5
+ # via apache-superset
+click-plugins==1.1.1
+ # via celery
+click-repl==0.2.0
+ # via celery
+colorama==0.4.6
+ # via
+ # apache-superset
+ # flask-appbuilder
+ # tox
+coverage[toml]==7.2.5
+ # via
+ # coverage
+ # pytest-cov
+cron-descriptor==1.2.24
+ # via apache-superset
+croniter==1.0.15
+ # via apache-superset
+cryptography==42.0.4
+ # via
+ # apache-superset
+ # paramiko
+ # pyjwt
+decorator==5.1.1
+ # via ipython
+delegator-py==0.1.1
+ # via cherrytree
+deprecated==1.2.13
+ # via
+ # limits
+ # pygithub
+deprecation==2.1.0
+ # via apache-superset
+dill==0.3.6
+ # via pylint
+distlib==0.3.6
+ # via virtualenv
+dnspython==2.1.0
+ # via email-validator
+docker==6.1.1
+ # via apache-superset
+email-validator==1.1.3
+ # via flask-appbuilder
+et-xmlfile==1.1.0
+ # via openpyxl
+exceptiongroup==1.2.0
+ # via
+ # cattrs
+ # pytest
+executing==1.2.0
+ # via stack-data
+filelock==3.12.2
+ # via
+ # tox
+ # virtualenv
+flask==2.2.5
+ # via
+ # apache-superset
+ # flask-appbuilder
+ # flask-babel
+ # flask-caching
+ # flask-compress
+ # flask-jwt-extended
+ # flask-limiter
+ # flask-login
+ # flask-migrate
+ # flask-session
+ # flask-sqlalchemy
+ # flask-testing
+ # flask-wtf
+flask-appbuilder==4.4.1
+ # via apache-superset
+flask-babel==1.0.0
+ # via flask-appbuilder
+flask-caching==2.1.0
+ # via apache-superset
+flask-compress==1.13
+ # via apache-superset
+flask-jwt-extended==4.3.1
+ # via flask-appbuilder
+flask-limiter==3.3.1
+ # via flask-appbuilder
+flask-login==0.6.3
+ # via
+ # apache-superset
+ # flask-appbuilder
+flask-migrate==3.1.0
+ # via apache-superset
+flask-session==0.5.0
+ # via apache-superset
+flask-sqlalchemy==2.5.1
+ # via
+ # flask-appbuilder
+ # flask-migrate
+flask-talisman==1.0.0
+ # via apache-superset
+flask-testing==0.8.1
+ # via apache-superset
+flask-wtf==1.2.1
+ # via
+ # apache-superset
+ # flask-appbuilder
+freezegun==1.2.2
+ # via apache-superset
+func-timeout==4.3.5
+ # via apache-superset
+future==0.18.3
+ # via pyhive
+geographiclib==1.52
+ # via geopy
+geopy==2.2.0
+ # via apache-superset
+gitdb==4.0.11
+ # via gitpython
+gitpython==3.1.42
+ # via cherrytree
+google-auth==2.27.0
+ # via shillelagh
+greenlet==3.0.3
+ # via
+ # apache-superset
+ # shillelagh
+grpcio==1.60.1
+ # via apache-superset
+gunicorn==21.2.0
+ # via apache-superset
+hashids==1.3.1
+ # via apache-superset
+holidays==0.25
+ # via apache-superset
+humanize==3.11.0
+ # via apache-superset
+identify==2.5.24
+ # via pre-commit
+idna==3.2
+ # via
+ # email-validator
+ # requests
+ijson==3.2.0.post0
+ # via tabulator
+importlib-metadata==6.6.0
+ # via
+ # apache-superset
+ # flask
+ # shillelagh
+importlib-resources==5.12.0
+ # via limits
+iniconfig==2.0.0
+ # via pytest
+ipython==8.12.2
+ # via apache-superset
+isodate==0.6.0
+ # via
+ # apache-superset
+ # tableschema
+isort==5.12.0
+ # via pylint
+itsdangerous==2.1.2
+ # via
+ # flask
+ # flask-wtf
+jedi==0.18.2
+ # via ipython
+jinja2==3.1.3
+ # via
+ # flask
+ # flask-babel
+jmespath==1.0.1
+ # via
+ # boto3
+ # botocore
+jsonlines==3.1.0
+ # via tabulator
+jsonschema==4.17.3
+ # via
+ # flask-appbuilder
+ # jsonschema-spec
+ # openapi-schema-validator
+ # openapi-spec-validator
+ # tableschema
+jsonschema-spec==0.1.4
+ # via openapi-spec-validator
+kombu==5.3.4
+ # via celery
+korean-lunar-calendar==0.3.1
+ # via holidays
+lazy-object-proxy==1.9.0
+ # via
+ # astroid
+ # openapi-spec-validator
+limits==3.4.0
+ # via flask-limiter
+linear-tsv==1.1.0
+ # via tabulator
+llvmlite==0.40.1
+ # via numba
+mako==1.2.4
+ # via
+ # alembic
+ # apache-superset
+markdown==3.3.4
+ # via apache-superset
+markdown-it-py==2.2.0
+ # via rich
+markupsafe==2.1.1
+ # via
+ # jinja2
+ # mako
+ # werkzeug
+ # wtforms
+marshmallow==3.19.0
+ # via
+ # flask-appbuilder
+ # marshmallow-sqlalchemy
+marshmallow-sqlalchemy==0.23.1
+ # via flask-appbuilder
+matplotlib-inline==0.1.6
+ # via ipython
+mccabe==0.7.0
+ # via pylint
+mdurl==0.1.2
+ # via markdown-it-py
+msgpack==1.0.2
+ # via apache-superset
+nh3==0.2.11
+ # via apache-superset
+nodeenv==1.7.0
+ # via pre-commit
+numba==0.57.1
+ # via pandas
+numexpr==2.9.0
+ # via pandas
+numpy==1.23.5
+ # via
+ # apache-superset
+ # bottleneck
+ # numba
+ # numexpr
+ # pandas
+ # pyarrow
+openapi-schema-validator==0.4.4
+ # via openapi-spec-validator
+openapi-spec-validator==0.5.6
+ # via apache-superset
+openpyxl==3.1.2
+ # via tabulator
+ordered-set==4.1.0
+ # via flask-limiter
+packaging==23.1
+ # via
+ # apache-superset
+ # apispec
+ # deprecation
+ # docker
+ # gunicorn
+ # limits
+ # marshmallow
+ # pyproject-api
+ # pytest
+ # shillelagh
+ # tox
+pandas[performance]==2.0.3
+ # via
+ # apache-superset
+ # pandas
+parameterized==0.9.0
+ # via apache-superset
+paramiko==3.4.0
+ # via
+ # apache-superset
+ # sshtunnel
+parsedatetime==2.6
+ # via apache-superset
+parso==0.8.3
+ # via jedi
+pathable==0.4.3
+ # via jsonschema-spec
+pexpect==4.8.0
+ # via
+ # delegator-py
+ # ipython
+pgsanity==0.2.9
+ # via apache-superset
+pickleshare==0.7.5
+ # via ipython
+platformdirs==3.8.1
+ # via
+ # pylint
+ # requests-cache
+ # tox
+ # virtualenv
+pluggy==1.2.0
+ # via
+ # pytest
+ # tox
+polyline==2.0.0
+ # via apache-superset
+pre-commit==3.3.3
+ # via apache-superset
+prison==0.2.1
+ # via flask-appbuilder
+progress==1.6
+ # via apache-superset
+prompt-toolkit==3.0.38
+ # via
+ # click-repl
+ # ipython
+ptyprocess==0.7.0
+ # via pexpect
+pure-eval==0.2.2
+ # via stack-data
+pure-sasl==0.6.2
+ # via
+ # pyhive
+ # thrift-sasl
+pyarrow==14.0.1
+ # via apache-superset
+pyasn1==0.5.1
+ # via
+ # pyasn1-modules
+ # python-ldap
+ # rsa
+pyasn1-modules==0.3.0
+ # via
+ # google-auth
+ # python-ldap
+pycparser==2.20
+ # via cffi
+pyfakefs==5.2.2
+ # via apache-superset
+pygithub==2.2.0
+ # via cherrytree
+pygments==2.15.0
+ # via
+ # ipython
+ # rich
+pyhive[hive,hive-pure-sasl]==0.7.0
+ # via
+ # apache-superset
+ # pyhive
+pyinstrument==4.4.0
+ # via apache-superset
+pyjwt[crypto]==2.4.0
+ # via
+ # apache-superset
+ # flask-appbuilder
+ # flask-jwt-extended
+ # pygithub
+pylint==2.17.7
+ # via apache-superset
+pynacl==1.5.0
+ # via
+ # paramiko
+ # pygithub
+pyparsing==3.0.6
+ # via apache-superset
+pyproject-api==1.5.2
+ # via tox
+pyrsistent==0.19.3
+ # via jsonschema
+pytest==7.3.1
+ # via
+ # apache-superset
+ # pytest-cov
+ # pytest-mock
+pytest-cov==4.0.0
+ # via apache-superset
+pytest-mock==3.10.0
+ # via apache-superset
+python-dateutil==2.8.2
+ # via
+ # alembic
+ # apache-superset
+ # botocore
+ # celery
+ # cherrytree
+ # croniter
+ # flask-appbuilder
+ # freezegun
+ # holidays
+ # pandas
+ # pyhive
+ # shillelagh
+ # tableschema
+python-dotenv==0.19.0
+ # via apache-superset
+python-editor==1.0.4
+ # via alembic
+python-geohash==0.8.5
+ # via apache-superset
+python-ldap==3.4.3
+ # via apache-superset
+pytz==2021.3
+ # via
+ # babel
+ # flask-babel
+ # pandas
+pyyaml==6.0.1
+ # via
+ # apache-superset
+ # apispec
+ # cherrytree
+ # jsonschema-spec
+ # pre-commit
+redis==4.6.0
+ # via apache-superset
+requests==2.31.0
+ # via
+ # docker
+ # pygithub
+ # requests-cache
+ # shillelagh
+ # tableschema
+ # tabulator
+requests-cache==1.1.1
+ # via shillelagh
+rfc3339-validator==0.1.4
+ # via openapi-schema-validator
+rfc3986==2.0.0
+ # via tableschema
+rich==13.3.4
+ # via flask-limiter
+rsa==4.9
+ # via google-auth
+s3transfer==0.8.0
+ # via boto3
+sasl==0.3.1
+ # via pyhive
+selenium==3.141.0
+ # via apache-superset
+shillelagh[gsheetsapi]==1.2.10
+ # via
+ # apache-superset
+ # shillelagh
+shortid==0.1.2
+ # via apache-superset
+simplejson==3.17.3
+ # via apache-superset
+six==1.16.0
+ # via
+ # asttokens
+ # click-repl
+ # isodate
+ # linear-tsv
+ # prison
+ # python-dateutil
+ # rfc3339-validator
+ # sasl
+ # tableschema
+ # tabulator
+ # thrift
+ # thrift-sasl
+ # url-normalize
+ # wtforms-json
+slack-sdk==3.21.3
+ # via apache-superset
+smmap==5.0.1
+ # via gitdb
+sqlalchemy==1.4.36
+ # via
+ # alembic
+ # apache-superset
+ # flask-appbuilder
+ # flask-sqlalchemy
+ # marshmallow-sqlalchemy
+ # shillelagh
+ # sqlalchemy-utils
+ # tabulator
+sqlalchemy-utils==0.38.3
+ # via
+ # apache-superset
+ # flask-appbuilder
+sqlglot==20.8.0
+ # via apache-superset
+sqloxide==0.1.33
+ # via apache-superset
+sqlparse==0.4.4
+ # via apache-superset
+sshtunnel==0.4.0
+ # via apache-superset
+stack-data==0.6.2
+ # via ipython
+statsd==4.0.1
+ # via apache-superset
+tableschema==1.20.2
+ # via apache-superset
+tabulate==0.8.9
+ # via apache-superset
+tabulator==1.53.5
+ # via tableschema
+termcolor==2.4.0
+ # via yaspin
+thrift==0.16.0
+ # via
+ # apache-superset
+ # pyhive
+ # thrift-sasl
+thrift-sasl==0.4.3
+ # via pyhive
+tomli==2.0.1
+ # via
+ # coverage
+ # pylint
+ # pyproject-api
+ # pytest
+ # tox
+tomlkit==0.11.8
+ # via pylint
+tox==4.6.4
+ # via apache-superset
+traitlets==5.9.0
+ # via
+ # ipython
+ # matplotlib-inline
+typing-extensions==4.4.0
+ # via
+ # apache-superset
+ # astroid
+ # cattrs
+ # flask-limiter
+ # ipython
+ # jsonschema-spec
+ # kombu
+ # limits
+ # pygithub
+ # pylint
+ # shillelagh
+tzdata==2023.3
+ # via
+ # celery
+ # pandas
+unicodecsv==0.14.1
+ # via
+ # tableschema
+ # tabulator
+url-normalize==1.4.3
+ # via requests-cache
+urllib3==1.26.18
+ # via
+ # botocore
+ # docker
+ # pygithub
+ # requests
+ # requests-cache
+ # selenium
+vine==5.1.0
+ # via
+ # amqp
+ # celery
+ # kombu
+virtualenv==20.23.1
+ # via
+ # pre-commit
+ # tox
+wcwidth==0.2.5
+ # via prompt-toolkit
+websocket-client==1.5.1
+ # via docker
+werkzeug==3.0.1
+ # via
+ # flask
+ # flask-appbuilder
+ # flask-jwt-extended
+ # flask-login
+wrapt==1.15.0
+ # via
+ # astroid
+ # deprecated
+wtforms==2.3.3
+ # via
+ # apache-superset
+ # flask-appbuilder
+ # flask-wtf
+ # wtforms-json
+wtforms-json==0.3.5
+ # via apache-superset
+xlrd==2.0.1
+ # via tabulator
+xlsxwriter==3.0.7
+ # via apache-superset
+yaspin==3.0.1
+ # via cherrytree
+zipp==3.15.0
+ # via
+ # importlib-metadata
+ # importlib-resources
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
(superset) 01/02: drying actions
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch kill-pip-compile-multi
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 012d2b5c512482a39baa9b5e4889108686616a5a
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Tue Mar 12 18:00:28 2024 -0700
drying actions
---
.../workflows/superset-python-integrationtest.yml | 28 +---
.github/workflows/superset-python-unittest.yml | 18 +--
pyproject.toml | 6 +
requirements/base.txt | 1 -
requirements/development.txt | 2 -
requirements/pip-compile-custom.sh | 32 -----
requirements/pip-compile-superset.py | 148 +++++++++++++++++++++
requirements/pip-compile-superset.sh | 33 -----
requirements/requirements-custom-example.in | 2 +-
9 files changed, 160 insertions(+), 110 deletions(-)
diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml
index 8f753e1668..da9136713d 100644
--- a/.github/workflows/superset-python-integrationtest.yml
+++ b/.github/workflows/superset-python-integrationtest.yml
@@ -51,22 +51,8 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- - name: Install dependencies
- if: steps.check.outcome == 'failure'
- uses: ./.github/actions/cached-dependencies
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install wheel
- pip install -r requirements/development.txt
- setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -117,12 +103,8 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
@@ -177,12 +159,8 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml
index 3730c59c03..309cade866 100644
--- a/.github/workflows/superset-python-unittest.yml
+++ b/.github/workflows/superset-python-unittest.yml
@@ -46,24 +46,10 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- # TODO: separated requirements.txt file just for unit tests
- - name: Install dependencies
- if: steps.check.outcome == 'failure'
- uses: ./.github/actions/cached-dependencies
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install wheel
- pip install -r requirements/development.txt
- pip install shillelagh
- mkdir ${{ github.workspace }}/.temp
+ requirements-type: 'dev'
- name: Python unit tests
if: steps.check.outcome == 'failure'
env:
diff --git a/pyproject.toml b/pyproject.toml
index 75e9ecaf7d..e6c3db4b8a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -205,3 +205,9 @@ development = [
"statsd",
"tox",
]
+
+[tool.mypy]
+exclude = '^requirements/*.py'
+
+[mypy-click]
+ignore_missing_imports = True
diff --git a/requirements/base.txt b/requirements/base.txt
index 6fb321fb24..a5ee1da927 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -405,4 +405,3 @@ zipp==3.15.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools
--e .
diff --git a/requirements/development.txt b/requirements/development.txt
index 71c8464d83..ea19e6401e 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -809,5 +809,3 @@ zope-interface==5.4.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools
--e .
--e .
diff --git a/requirements/pip-compile-custom.sh b/requirements/pip-compile-custom.sh
deleted file mode 100644
index bc04d19b33..0000000000
--- a/requirements/pip-compile-custom.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A simple example of how to customize your python dependencies while re-using
-# as much as the pinned ones from the true-and-tested ones
-
-# first, create your own `requirements.in` file, as this example file
-# see `requirements/requirements-custom-example.in` as an example
-
-# copy the pinned dependency so that we can make it the target outpout for pip-compile
-# here we're getting pip-compile into using its output as an input
-cp requirements/development.txt requirements/requirements-custom-example.txt
-
-# pip-compile mixing your input and output (which also acts as the basis)
-pip-compile -o requirements-custom-example.in requirements-custom-example.in
-
-# this ideally is done as part of the release process, whenever any
-# of the files referenced here change, including requirements/development.txt
diff --git a/requirements/pip-compile-superset.py b/requirements/pip-compile-superset.py
new file mode 100644
index 0000000000..9065004a94
--- /dev/null
+++ b/requirements/pip-compile-superset.py
@@ -0,0 +1,148 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import subprocess
+import sys
+from shutil import copyfile
+
+import click
+
+BASE_REQS = "requirements/base.txt"
+DEV_REQS = "requirements/development.txt"
+
+DEV_EXTRAS = [
+ "bigquery",
+ "cors",
+ "development",
+ "druid",
+ "hive",
+ "gevent",
+ "mysql",
+ "postgres",
+ "presto",
+ "prophet",
+ "trino",
+ "gsheets",
+ "playwright",
+ "thumbnails",
+]
+
+
+def read_requirements(path):
+ """Read requirements from a file and return them as a dictionary."""
+ requirements = {}
+ with open(path) as file:
+ for line in file:
+ line = line.strip()
+ if line and not line.startswith("#"):
+ parts = line.split("==")
+ lib_name = parts[0].strip()
+ version = parts[1].strip() if len(parts) == 2 else None
+ requirements[lib_name] = version
+ return requirements
+
+
+def compare_requirements(reqs1, reqs2):
+ """Compare two sets of requirements and identify differences."""
+ added = {lib: ver for lib, ver in reqs2.items() if lib not in reqs1}
+ removed = {lib: ver for lib, ver in reqs1.items() if lib not in reqs2}
+ version_changed = {
+ lib: (reqs1[lib], reqs2[lib])
+ for lib in reqs1
+ if lib in reqs2 and reqs1[lib] != reqs2[lib]
+ }
+ return added, removed, version_changed
+
+
+def bash(cmd):
+ print(f"RUN: {cmd}")
+ result = subprocess.run(cmd, shell=True)
+ if result.returncode != 0:
+ print(f"Error: Command '{cmd}' exited with {result.returncode}")
+ sys.exit(result.returncode)
+
+
+@click.group()
+def cli():
+ pass
+
+
+@click.command()
+@click.option("--pip-flags", default="", help="Flags to pass directly to pip-compile.")
+def compile_deps(pip_flags):
+ """Compile dependencies using pip-compile with optional flags."""
+ # pip-compile commands
+ bash(f"pip-compile -o {BASE_REQS} {pip_flags}")
+ bash(f'pip-compile -o {DEV_REQS} -v {pip_flags} --extra {",".join(DEV_EXTRAS)}')
+
+ click.echo("Dependencies compiled.")
+
+
+@click.command()
+@click.option(
+ "--dev", is_flag=True, help="Install development dependencies instead of base."
+)
+def install_deps(dev):
+ """Install dependencies from the compiled requirements file."""
+ file_path = "requirements/development.txt" if dev else "requirements/base.txt"
+ bash(f"pip install -r {file_path}")
+ # Installing the project itself in editable mode
+ bash("pip install -e .")
+ click.echo(f"Dependencies from {file_path} and project installed.")
+
+
+@click.command()
+@click.argument("file1", type=click.Path(exists=True), default=BASE_REQS)
+@click.argument("file2", type=click.Path(exists=True), default=DEV_REQS)
+def compare_versions(file1, file2):
+ """Load two requirements files, compare them, and print differences."""
+ reqs1 = read_requirements(file1)
+ reqs2 = read_requirements(file2)
+ added, removed, version_changed = compare_requirements(reqs1, reqs2)
+
+ if added:
+ click.echo("Added:")
+ for lib, ver in added.items():
+ click.echo(f"{lib}=={ver}")
+ if removed:
+ click.echo("\nRemoved:")
+ for lib, ver in removed.items():
+ click.echo(f"{lib}=={ver}")
+ if version_changed:
+ click.echo("\nVersion Changed:")
+ for lib, versions in version_changed.items():
+ click.echo(f"{lib}: from {versions[0]} to {versions[1]}")
+
+
+@click.command()
+@click.argument("input_file")
+@click.argument("output_file")
+def merge_compile(input_file, output_file):
+ """Merge, compile and check versions."""
+ # Step 1: Copy development.txt to the output file
+ copyfile(DEV_REQS, output_file)
+
+ # Step 2 & 3: Compile the user-defined requirements file appending to the output
+ bash(f"pip-compile -v {input_file} --output-file={output_file}")
+
+
+cli.add_command(merge_compile)
+cli.add_command(compile_deps)
+cli.add_command(install_deps)
+cli.add_command(compare_versions)
+
+if __name__ == "__main__":
+ cli()
diff --git a/requirements/pip-compile-superset.sh b/requirements/pip-compile-superset.sh
deleted file mode 100755
index 34f950ca42..0000000000
--- a/requirements/pip-compile-superset.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A simple bash script to "compile"/pin our python dependencies using pip-compile
-
-# Let's forward all script arguments to pip-compile
-# you can pass things like `--no-upgrade` or target a specific package with `-P "flask"`
-pip_compile_flags="$@"
-
-# Compile the base requirements
-pip-compile -o requirements/base.txt $pip_compile_flags
-
-# Compile the development requirements with extras
-pip-compile -o requirements/development.txt -v $pip_compile_flags \
- --extra bigquery,cors,development,druid,hive,gevent,mysql,postgres,presto,prophet,trino,gsheets,playwright,shillelagh,thumbnails
-
-# Append '-e .' to both requirements files to include the project itself
-echo "-e ." >> requirements/base.txt
-echo "-e ." >> requirements/development.txt
diff --git a/requirements/requirements-custom-example.in b/requirements/requirements-custom-example.in
index 4d1b630dfe..215256ab9f 100644
--- a/requirements/requirements-custom-example.in
+++ b/requirements/requirements-custom-example.in
@@ -5,4 +5,4 @@
cherrytree
# pin an existing Superset reference to a specific version
-boto3==1.18.44
+boto3==1.29.7