You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ar...@apache.org on 2022/05/17 22:29:02 UTC
[tvm] 25/34: Adjust Dockerfile python installs
This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch areusch/freeze-dependencies
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 7de708d03081229ddd811468a847e59a816818be
Author: Andrew Reusch <ar...@gmail.com>
AuthorDate: Fri May 13 14:02:06 2022 -0700
Adjust Dockerfile python installs
---
docker/Dockerfile.ci_arm | 3 ++
docker/Dockerfile.ci_gpu | 6 ++--
docker/Dockerfile.ci_hexagon | 6 ++--
docker/Dockerfile.ci_i386 | 3 ++
docker/Dockerfile.ci_lint | 9 +++---
docker/Dockerfile.ci_qemu | 8 ++---
docker/Dockerfile.ci_wasm | 3 ++
docker/install/ubuntu1804_install_python_venv.sh | 31 -------------------
docker/install/ubuntu2004_install_python.sh | 38 +++++++++++++++++++-----
9 files changed, 54 insertions(+), 53 deletions(-)
diff --git a/docker/Dockerfile.ci_arm b/docker/Dockerfile.ci_arm
index c19f1ff5a4..42f9ebe705 100644
--- a/docker/Dockerfile.ci_arm
+++ b/docker/Dockerfile.ci_arm
@@ -44,6 +44,7 @@ ENV PATH /opt/sccache:$PATH
COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
RUN bash /install/ubuntu_install_llvm.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
@@ -53,6 +54,8 @@ RUN pip config set global.no-cache-dir false
COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu
index 5a7db5315f..62d6eb2dbe 100644
--- a/docker/Dockerfile.ci_gpu
+++ b/docker/Dockerfile.ci_gpu
@@ -33,12 +33,10 @@ RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
-
COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh
@@ -48,6 +46,8 @@ RUN bash /install/ubuntu1804_install_llvm.sh
COPY install/ubuntu_install_opencl.sh /install/ubuntu_install_opencl.sh
RUN bash /install/ubuntu_install_opencl.sh
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon
index 20b185ab64..0057002f9e 100644
--- a/docker/Dockerfile.ci_hexagon
+++ b/docker/Dockerfile.ci_hexagon
@@ -28,12 +28,10 @@ RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh
RUN bash /install/ubuntu2004_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.cache-dir false
-
# Rust env (build early; takes a while)
COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
RUN bash /install/ubuntu_install_rust.sh
@@ -41,6 +39,8 @@ ENV RUSTUP_HOME /opt/rust
ENV CARGO_HOME /opt/rust
ENV PATH $PATH:$CARGO_HOME/bin
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386
index bbf449d9c3..4a0b359836 100644
--- a/docker/Dockerfile.ci_i386
+++ b/docker/Dockerfile.ci_i386
@@ -31,6 +31,7 @@ RUN bash /install/ubuntu_install_googletest.sh
COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
RUN bash /install/ubuntu_install_llvm.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
@@ -40,6 +41,8 @@ RUN pip config set global.no-cache-dir false
COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_lint b/docker/Dockerfile.ci_lint
index 1d0c984c61..38e3567a78 100644
--- a/docker/Dockerfile.ci_lint
+++ b/docker/Dockerfile.ci_lint
@@ -24,16 +24,17 @@ RUN apt-get update --fix-missing
RUN apt-get update && apt-get install -y wget git sudo make parallel
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
RUN apt-get update && apt-get install -y doxygen graphviz curl shellcheck
-RUN pip3 install cpplint pylint==2.4.4 mypy==0.902 black==22.3.0 flake8==3.9.2 blocklint==0.2.3 jinja2==3.0.3
-
# Rust env (build early; takes a while)
COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
RUN bash /install/ubuntu_install_rust.sh
diff --git a/docker/Dockerfile.ci_qemu b/docker/Dockerfile.ci_qemu
index 28bfd8962d..6082139abc 100644
--- a/docker/Dockerfile.ci_qemu
+++ b/docker/Dockerfile.ci_qemu
@@ -27,16 +27,15 @@ RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
COPY install/ubuntu1804_install_python_venv.sh /install/ubuntu1804_install_python_venv.sh
RUN bash /install/ubuntu1804_install_python_venv.sh
-ENV PATH=/opt/tvm-venv/bin:/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
@@ -77,6 +76,7 @@ COPY install/ubuntu_init_zephyr_project.sh /install/ubuntu_init_zephyr_project.s
COPY install/ubuntu_install_zephyr_sdk.sh /install/ubuntu_install_zephyr_sdk.sh
RUN bash /install/ubuntu_install_zephyr.sh
ENV ZEPHYR_BASE=/opt/zephyrproject/zephyr
+ENV PATH=/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH
# FreeRTOS deps
COPY install/ubuntu_install_freertos.sh /install/ubuntu_install_freertos.sh
diff --git a/docker/Dockerfile.ci_wasm b/docker/Dockerfile.ci_wasm
index 1c7d3eb59b..5868e87fef 100644
--- a/docker/Dockerfile.ci_wasm
+++ b/docker/Dockerfile.ci_wasm
@@ -24,12 +24,15 @@ RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
# Globally disable pip cache
RUN pip config set global.no-cache-dir false
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/install/ubuntu1804_install_python_venv.sh b/docker/install/ubuntu1804_install_python_venv.sh
deleted file mode 100755
index 5dc5efea76..0000000000
--- a/docker/install/ubuntu1804_install_python_venv.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-# install python and pip, don't modify this, modify install_python_package.sh
-apt-get update
-apt-get install -y software-properties-common
-apt-get install -y python3.7-dev python3-setuptools python3.7-venv
-
-python3 -mvenv /opt/tvm-venv
-
-# Pin pip and setuptools versions
-/opt/tvm-venv/bin/pip3 install pip==19.3.1 setuptools==58.4.0
diff --git a/docker/install/ubuntu2004_install_python.sh b/docker/install/ubuntu2004_install_python.sh
index 5b87a74061..31f803a90f 100755
--- a/docker/install/ubuntu2004_install_python.sh
+++ b/docker/install/ubuntu2004_install_python.sh
@@ -34,12 +34,34 @@ apt-get install -y software-properties-common
apt-get install -y python3.8 python3.8-dev python3-pip
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
-# Pin pip and setuptools versions
-# Hashes generated via:
-# $ pip download <package>==<version>
-# $ pip hash --algorithm sha256 <package>.whl
-cat <<EOF > base-requirements.txt
-pip==22.0.4 --hash=sha256:c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b
-setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036
+function download_hash() {
+ cat >/tmp/hash-bootstrap-packages.py <<EOF
+import os
+import os.path
+import subprocess
+import pkginfo
+
+for f in sorted(os.scandir("."), key=lambda x: x.name):
+ if not f.is_file():
+ continue
+ p = pkginfo.get_metadata(f.name)
+ if not p:
+ continue
+ print(f"{p.name}=={p.version} {subprocess.check_output(['pip3', 'hash', '-a', 'sha256', p.filename], encoding='utf-8').split()[1]} # {f.name}")
EOF
-pip3 install -r base-requirements.txt
+ mkdir packages && cd packages
+ pip3 install -U "$@"
+ pip3 download pip poetry setuptools
+ python3 /tmp/hash-bootstrap-packages.py
+ exit 2 # make docker build stop
+}
+
+# Install bootstrap packages. You can update these with the following procedure:
+# 1. Uncomment the line below, then attempt torebuild the base images (it will fail).
+# 2. New hashes should be printed in the terminal log from each docker build. Copy these hashes into the
+# the arch-appropriate file in docker/python/bootstrap-requirements/
+# download_hash pip setuptools pkginfo
+
+pip3 install -U pip -c /install/python/bootstrap-requirements.txt # Update pip to match version used to produce base-requirements.txt
+pip3 config set global.no-cache-dir false
+pip3 install -r /install/python/bootstrap-requirements.txt -c /install/python/bootstrap-requirements.txt