You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/03/13 01:01:01 UTC
(superset) 01/02: drying actions
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch kill-pip-compile-multi
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 012d2b5c512482a39baa9b5e4889108686616a5a
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Tue Mar 12 18:00:28 2024 -0700
drying actions
---
.../workflows/superset-python-integrationtest.yml | 28 +---
.github/workflows/superset-python-unittest.yml | 18 +--
pyproject.toml | 6 +
requirements/base.txt | 1 -
requirements/development.txt | 2 -
requirements/pip-compile-custom.sh | 32 -----
requirements/pip-compile-superset.py | 148 +++++++++++++++++++++
requirements/pip-compile-superset.sh | 33 -----
requirements/requirements-custom-example.in | 2 +-
9 files changed, 160 insertions(+), 110 deletions(-)
diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml
index 8f753e1668..da9136713d 100644
--- a/.github/workflows/superset-python-integrationtest.yml
+++ b/.github/workflows/superset-python-integrationtest.yml
@@ -51,22 +51,8 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- - name: Install dependencies
- if: steps.check.outcome == 'failure'
- uses: ./.github/actions/cached-dependencies
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install wheel
- pip install -r requirements/development.txt
- setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -117,12 +103,8 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
@@ -177,12 +159,8 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml
index 3730c59c03..309cade866 100644
--- a/.github/workflows/superset-python-unittest.yml
+++ b/.github/workflows/superset-python-unittest.yml
@@ -46,24 +46,10 @@ jobs:
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
+ uses: ./.github/actions/setup-python.yml
if: steps.check.outcome == 'failure'
- uses: actions/setup-python@v5
with:
- python-version: ${{ matrix.python-version }}
- cache: "pip"
- cache-dependency-path: "requirements/development.txt"
- # TODO: separated requirements.txt file just for unit tests
- - name: Install dependencies
- if: steps.check.outcome == 'failure'
- uses: ./.github/actions/cached-dependencies
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install wheel
- pip install -r requirements/development.txt
- pip install shillelagh
- mkdir ${{ github.workspace }}/.temp
+ requirements-type: 'dev'
- name: Python unit tests
if: steps.check.outcome == 'failure'
env:
diff --git a/pyproject.toml b/pyproject.toml
index 75e9ecaf7d..e6c3db4b8a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -205,3 +205,9 @@ development = [
"statsd",
"tox",
]
+
+[tool.mypy]
+exclude = '^requirements/*.py'
+
+[mypy-click]
+ignore_missing_imports = True
diff --git a/requirements/base.txt b/requirements/base.txt
index 6fb321fb24..a5ee1da927 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -405,4 +405,3 @@ zipp==3.15.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools
--e .
diff --git a/requirements/development.txt b/requirements/development.txt
index 71c8464d83..ea19e6401e 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -809,5 +809,3 @@ zope-interface==5.4.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools
--e .
--e .
diff --git a/requirements/pip-compile-custom.sh b/requirements/pip-compile-custom.sh
deleted file mode 100644
index bc04d19b33..0000000000
--- a/requirements/pip-compile-custom.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A simple example of how to customize your python dependencies while re-using
-# as much as the pinned ones from the true-and-tested ones
-
-# first, create your own `requirements.in` file, as this example file
-# see `requirements/requirements-custom-example.in` as an example
-
-# copy the pinned dependency so that we can make it the target outpout for pip-compile
-# here we're getting pip-compile into using its output as an input
-cp requirements/development.txt requirements/requirements-custom-example.txt
-
-# pip-compile mixing your input and output (which also acts as the basis)
-pip-compile -o requirements-custom-example.in requirements-custom-example.in
-
-# this ideally is done as part of the release process, whenever any
-# of the files referenced here change, including requirements/development.txt
diff --git a/requirements/pip-compile-superset.py b/requirements/pip-compile-superset.py
new file mode 100644
index 0000000000..9065004a94
--- /dev/null
+++ b/requirements/pip-compile-superset.py
@@ -0,0 +1,148 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import subprocess
+import sys
+from shutil import copyfile
+
+import click
+
+BASE_REQS = "requirements/base.txt"
+DEV_REQS = "requirements/development.txt"
+
+DEV_EXTRAS = [
+ "bigquery",
+ "cors",
+ "development",
+ "druid",
+ "hive",
+ "gevent",
+ "mysql",
+ "postgres",
+ "presto",
+ "prophet",
+ "trino",
+ "gsheets",
+ "playwright",
+ "thumbnails",
+]
+
+
+def read_requirements(path):
+ """Read requirements from a file and return them as a dictionary."""
+ requirements = {}
+ with open(path) as file:
+ for line in file:
+ line = line.strip()
+ if line and not line.startswith("#"):
+ parts = line.split("==")
+ lib_name = parts[0].strip()
+ version = parts[1].strip() if len(parts) == 2 else None
+ requirements[lib_name] = version
+ return requirements
+
+
+def compare_requirements(reqs1, reqs2):
+ """Compare two sets of requirements and identify differences."""
+ added = {lib: ver for lib, ver in reqs2.items() if lib not in reqs1}
+ removed = {lib: ver for lib, ver in reqs1.items() if lib not in reqs2}
+ version_changed = {
+ lib: (reqs1[lib], reqs2[lib])
+ for lib in reqs1
+ if lib in reqs2 and reqs1[lib] != reqs2[lib]
+ }
+ return added, removed, version_changed
+
+
+def bash(cmd):
+ print(f"RUN: {cmd}")
+ result = subprocess.run(cmd, shell=True)
+ if result.returncode != 0:
+ print(f"Error: Command '{cmd}' exited with {result.returncode}")
+ sys.exit(result.returncode)
+
+
+@click.group()
+def cli():
+ pass
+
+
+@click.command()
+@click.option("--pip-flags", default="", help="Flags to pass directly to pip-compile.")
+def compile_deps(pip_flags):
+ """Compile dependencies using pip-compile with optional flags."""
+ # pip-compile commands
+ bash(f"pip-compile -o {BASE_REQS} {pip_flags}")
+ bash(f'pip-compile -o {DEV_REQS} -v {pip_flags} --extra {",".join(DEV_EXTRAS)}')
+
+ click.echo("Dependencies compiled.")
+
+
+@click.command()
+@click.option(
+ "--dev", is_flag=True, help="Install development dependencies instead of base."
+)
+def install_deps(dev):
+ """Install dependencies from the compiled requirements file."""
+ file_path = "requirements/development.txt" if dev else "requirements/base.txt"
+ bash(f"pip install -r {file_path}")
+ # Installing the project itself in editable mode
+ bash("pip install -e .")
+ click.echo(f"Dependencies from {file_path} and project installed.")
+
+
+@click.command()
+@click.argument("file1", type=click.Path(exists=True), default=BASE_REQS)
+@click.argument("file2", type=click.Path(exists=True), default=DEV_REQS)
+def compare_versions(file1, file2):
+ """Load two requirements files, compare them, and print differences."""
+ reqs1 = read_requirements(file1)
+ reqs2 = read_requirements(file2)
+ added, removed, version_changed = compare_requirements(reqs1, reqs2)
+
+ if added:
+ click.echo("Added:")
+ for lib, ver in added.items():
+ click.echo(f"{lib}=={ver}")
+ if removed:
+ click.echo("\nRemoved:")
+ for lib, ver in removed.items():
+ click.echo(f"{lib}=={ver}")
+ if version_changed:
+ click.echo("\nVersion Changed:")
+ for lib, versions in version_changed.items():
+ click.echo(f"{lib}: from {versions[0]} to {versions[1]}")
+
+
+@click.command()
+@click.argument("input_file")
+@click.argument("output_file")
+def merge_compile(input_file, output_file):
+ """Merge, compile and check versions."""
+ # Step 1: Copy development.txt to the output file
+ copyfile(DEV_REQS, output_file)
+
+ # Step 2 & 3: Compile the user-defined requirements file appending to the output
+ bash(f"pip-compile -v {input_file} --output-file={output_file}")
+
+
+cli.add_command(merge_compile)
+cli.add_command(compile_deps)
+cli.add_command(install_deps)
+cli.add_command(compare_versions)
+
+if __name__ == "__main__":
+ cli()
diff --git a/requirements/pip-compile-superset.sh b/requirements/pip-compile-superset.sh
deleted file mode 100755
index 34f950ca42..0000000000
--- a/requirements/pip-compile-superset.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A simple bash script to "compile"/pin our python dependencies using pip-compile
-
-# Let's forward all script arguments to pip-compile
-# you can pass things like `--no-upgrade` or target a specific package with `-P "flask"`
-pip_compile_flags="$@"
-
-# Compile the base requirements
-pip-compile -o requirements/base.txt $pip_compile_flags
-
-# Compile the development requirements with extras
-pip-compile -o requirements/development.txt -v $pip_compile_flags \
- --extra bigquery,cors,development,druid,hive,gevent,mysql,postgres,presto,prophet,trino,gsheets,playwright,shillelagh,thumbnails
-
-# Append '-e .' to both requirements files to include the project itself
-echo "-e ." >> requirements/base.txt
-echo "-e ." >> requirements/development.txt
diff --git a/requirements/requirements-custom-example.in b/requirements/requirements-custom-example.in
index 4d1b630dfe..215256ab9f 100644
--- a/requirements/requirements-custom-example.in
+++ b/requirements/requirements-custom-example.in
@@ -5,4 +5,4 @@
cherrytree
# pin an existing Superset reference to a specific version
-boto3==1.18.44
+boto3==1.29.7