You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ar...@apache.org on 2022/05/23 23:11:53 UTC
[tvm] 01/04: Align Python and package install process in all containers.
This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch areusch/freeze-dependencies
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 5f5c3c85b4f665ddb82dbaf0e1b396f592083809
Author: Andrew Reusch <ar...@gmail.com>
AuthorDate: Thu May 19 15:16:15 2022 -0700
Align Python and package install process in all containers.
---
docker/Dockerfile.ci_arm | 3 +
docker/Dockerfile.ci_cpu | 47 ++++++++-----
docker/Dockerfile.ci_gpu | 44 ++++++------
docker/Dockerfile.ci_hexagon | 12 ++--
docker/Dockerfile.ci_i386 | 15 +---
docker/Dockerfile.ci_lint | 13 ++--
docker/Dockerfile.ci_qemu | 13 ++--
docker/Dockerfile.ci_wasm | 8 +--
docker/install/ubuntu1804_install_python_venv.sh | 31 ---------
docker/install/ubuntu2004_install_python.sh | 38 ++++++++---
docker/install/ubuntu_install_caffe.sh | 3 -
docker/install/ubuntu_install_core.sh | 2 +-
docker/install/ubuntu_install_coreml.sh | 23 -------
docker/install/ubuntu_install_darknet.sh | 29 --------
docker/install/ubuntu_install_mxnet.sh | 23 -------
docker/install/ubuntu_install_oneflow.sh | 25 -------
docker/install/ubuntu_install_onnx.sh | 41 -----------
docker/install/ubuntu_install_paddle.sh | 23 -------
docker/install/ubuntu_install_python_package.sh | 36 +++-------
docker/install/ubuntu_install_redis.sh | 2 -
docker/install/ubuntu_install_sphinx.sh | 33 ---------
docker/install/ubuntu_install_tensorflow.sh | 26 -------
.../install/ubuntu_install_tensorflow_aarch64.sh | 8 ---
docker/install/ubuntu_install_tflite.sh | 79 ----------------------
docker/install/ubuntu_install_vela.sh | 23 -------
25 files changed, 117 insertions(+), 483 deletions(-)
diff --git a/docker/Dockerfile.ci_arm b/docker/Dockerfile.ci_arm
index c19f1ff5a4..42f9ebe705 100644
--- a/docker/Dockerfile.ci_arm
+++ b/docker/Dockerfile.ci_arm
@@ -44,6 +44,7 @@ ENV PATH /opt/sccache:$PATH
COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
RUN bash /install/ubuntu_install_llvm.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
@@ -53,6 +54,8 @@ RUN pip config set global.no-cache-dir false
COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu
index 45943334a0..d2bcc09da6 100644
--- a/docker/Dockerfile.ci_cpu
+++ b/docker/Dockerfile.ci_cpu
@@ -18,22 +18,35 @@
# CI docker CPU env
FROM ubuntu:18.04
-RUN apt-get update --fix-missing
-
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
-
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
-RUN bash /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh \
+ -E ethosu \
+ -E importer-caffe \
+ -E importer-caffe2 \
+ -E importer-coreml \
+ -E importer-darknet \
+ -E importer-keras \
+ -E importer-oneflow \
+ -E importer-onnx \
+ -E importer-paddle \
+ -E importer-pytorch \
+ -E importer-tensorflow \
+ -E importer-tflite \
+ -E tvmc \
+ -E xgboost \
+ ;
COPY install/ubuntu1804_install_llvm.sh /install/ubuntu1804_install_llvm.sh
RUN bash /install/ubuntu1804_install_llvm.sh
@@ -44,9 +57,9 @@ RUN bash /install/ubuntu_install_dnnl.sh
COPY install/ubuntu_install_papi.sh /install/ubuntu_install_papi.sh
RUN bash /install/ubuntu_install_papi.sh ""
-# Install MxNet for access to Gluon Model Zoo.
-COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh
-RUN bash /install/ubuntu_install_mxnet.sh
+# # Install MxNet for access to Gluon Model Zoo.
+# COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh
+# RUN bash /install/ubuntu_install_mxnet.sh
# Rust env (build early; takes a while)
COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
@@ -89,12 +102,12 @@ COPY install/ubuntu_install_verilator.sh /install/ubuntu_install_verilator.sh
RUN bash /install/ubuntu_install_verilator.sh
# TensorFlow deps
-COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh
-RUN bash /install/ubuntu_install_tensorflow.sh
+# COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh
+# RUN bash /install/ubuntu_install_tensorflow.sh
# TFLite deps
-COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh
-RUN bash /install/ubuntu_install_tflite.sh
+# COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh
+# RUN bash /install/ubuntu_install_tflite.sh
# Compute Library
COPY install/ubuntu_download_arm_compute_lib_binaries.sh /install/ubuntu_download_arm_compute_lib_binaries.sh
@@ -132,15 +145,15 @@ COPY install/ubuntu_install_ethosu_driver_stack.sh /install/ubuntu_install_ethos
RUN bash /install/ubuntu_install_ethosu_driver_stack.sh
# Install Vela compiler
-COPY install/ubuntu_install_vela.sh /install/ubuntu_install_vela.sh
-RUN bash /install/ubuntu_install_vela.sh
+# COPY install/ubuntu_install_vela.sh /install/ubuntu_install_vela.sh
+# RUN bash /install/ubuntu_install_vela.sh
# Update PATH
ENV PATH /opt/arm/gcc-arm-none-eabi/bin:/opt/arm/FVP_Corstone_SSE-300/models/Linux64_GCC-6.4:$PATH
# PaddlePaddle deps
-COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh
-RUN bash /install/ubuntu_install_paddle.sh
+# COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh
+# RUN bash /install/ubuntu_install_paddle.sh
# sccache
COPY install/ubuntu_install_sccache.sh /install/ubuntu_install_sccache.sh
diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu
index 73d13007f1..4be46d4821 100644
--- a/docker/Dockerfile.ci_gpu
+++ b/docker/Dockerfile.ci_gpu
@@ -25,7 +25,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/
# Base scripts
RUN rm /etc/apt/sources.list.d/nvidia-ml.list && apt-get clean
-RUN apt-get update --fix-missing
+# NOTE: apt-get update --fix-missing is run by ubuntu_install_core.sh.
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
@@ -33,12 +33,10 @@ RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
-
COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh
@@ -48,11 +46,13 @@ RUN bash /install/ubuntu1804_install_llvm.sh
COPY install/ubuntu_install_opencl.sh /install/ubuntu_install_opencl.sh
RUN bash /install/ubuntu_install_opencl.sh
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
-COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
-RUN bash /install/ubuntu_install_sphinx.sh
+# COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
+# RUN bash /install/ubuntu_install_sphinx.sh
# Enable doxygen for c++ doc build
RUN apt-get update && apt-get install -y doxygen libprotobuf-dev protobuf-compiler
@@ -67,29 +67,29 @@ COPY install/ubuntu_install_rocm.sh /install/ubuntu_install_rocm.sh
RUN bash /install/ubuntu_install_rocm.sh
# DL Frameworks
-COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh
-RUN bash /install/ubuntu_install_mxnet.sh
+# COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh
+# RUN bash /install/ubuntu_install_mxnet.sh
COPY install/ubuntu_install_gluoncv.sh /install/ubuntu_install_gluoncv.sh
RUN bash /install/ubuntu_install_gluoncv.sh
-COPY install/ubuntu_install_coreml.sh /install/ubuntu_install_coreml.sh
-RUN bash /install/ubuntu_install_coreml.sh
+# COPY install/ubuntu_install_coreml.sh /install/ubuntu_install_coreml.sh
+# RUN bash /install/ubuntu_install_coreml.sh
-COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh
-RUN bash /install/ubuntu_install_tensorflow.sh
+# COPY install/ubuntu_install_tensorflow.sh /install/ubuntu_install_tensorflow.sh
+# RUN bash /install/ubuntu_install_tensorflow.sh
-COPY install/ubuntu_install_darknet.sh /install/ubuntu_install_darknet.sh
-RUN bash /install/ubuntu_install_darknet.sh
+# COPY install/ubuntu_install_darknet.sh /install/ubuntu_install_darknet.sh
+# RUN bash /install/ubuntu_install_darknet.sh
-COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh
-RUN bash /install/ubuntu_install_onnx.sh
+# COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh
+# RUN bash /install/ubuntu_install_onnx.sh
COPY install/ubuntu_install_libtorch.sh /install/ubuntu_install_libtorch.sh
RUN bash /install/ubuntu_install_libtorch.sh
-COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh
-RUN bash /install/ubuntu_install_tflite.sh
+# COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh
+# RUN bash /install/ubuntu_install_tflite.sh
COPY install/ubuntu_install_dgl.sh /install/ubuntu_install_dgl.sh
RUN bash /install/ubuntu_install_dgl.sh
@@ -98,12 +98,12 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility
COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh
RUN bash /install/ubuntu_install_vulkan.sh
-COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh
-RUN bash /install/ubuntu_install_paddle.sh
+# COPY install/ubuntu_install_paddle.sh /install/ubuntu_install_paddle.sh
+# RUN bash /install/ubuntu_install_paddle.sh
# OneFlow deps
-COPY install/ubuntu_install_oneflow.sh /install/ubuntu_install_oneflow.sh
-RUN bash /install/ubuntu_install_oneflow.sh
+# COPY install/ubuntu_install_oneflow.sh /install/ubuntu_install_oneflow.sh
+# RUN bash /install/ubuntu_install_oneflow.sh
# Rust env (build early; takes a while)
COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon
index ddca5c6c2e..8115a27e14 100644
--- a/docker/Dockerfile.ci_hexagon
+++ b/docker/Dockerfile.ci_hexagon
@@ -19,21 +19,19 @@
# tag: v0.02
FROM tvmcihexagon/ci-hexagon-base:v0.02_SDK4.5.0.3
-RUN apt-get update --fix-missing
-RUN apt-get install -y ca-certificates gnupg2 libxml2-dev
-
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
+# TODO: why do we need this?
+# RUN apt-get install -y ca-certificates gnupg2 libxml2-dev
+
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh
RUN bash /install/ubuntu2004_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.cache-dir false
-
# Rust env (build early; takes a while)
COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
RUN bash /install/ubuntu_install_rust.sh
@@ -41,6 +39,8 @@ ENV RUSTUP_HOME /opt/rust
ENV CARGO_HOME /opt/rust
ENV PATH $PATH:$CARGO_HOME/bin
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386
index 61ba064ff3..b40bb06fdd 100644
--- a/docker/Dockerfile.ci_i386
+++ b/docker/Dockerfile.ci_i386
@@ -20,8 +20,6 @@
FROM i386/ubuntu:18.04
-RUN apt-get update --fix-missing && apt-get install -y ca-certificates
-
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
@@ -31,22 +29,15 @@ RUN bash /install/ubuntu_install_googletest.sh
COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
RUN bash /install/ubuntu_install_llvm.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Rust env (build early; takes a while)
-COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
-RUN bash /install/ubuntu_install_rust.sh
-ENV RUSTUP_HOME /opt/rust
-ENV CARGO_HOME /opt/rust
-ENV PATH $PATH:$CARGO_HOME/bin
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
-
COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/Dockerfile.ci_lint b/docker/Dockerfile.ci_lint
index 1d0c984c61..20d1699a88 100644
--- a/docker/Dockerfile.ci_lint
+++ b/docker/Dockerfile.ci_lint
@@ -20,20 +20,19 @@
# tag: v0.60
FROM ubuntu:18.04
-RUN apt-get update --fix-missing
-
-RUN apt-get update && apt-get install -y wget git sudo make parallel
+RUN apt-get update --fix-missing && apt-get install -y wget git sudo make parallel
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
RUN apt-get update && apt-get install -y doxygen graphviz curl shellcheck
-RUN pip3 install cpplint pylint==2.4.4 mypy==0.902 black==22.3.0 flake8==3.9.2 blocklint==0.2.3 jinja2==3.0.3
-
# Rust env (build early; takes a while)
COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
RUN bash /install/ubuntu_install_rust.sh
diff --git a/docker/Dockerfile.ci_qemu b/docker/Dockerfile.ci_qemu
index 28bfd8962d..13a86683e9 100644
--- a/docker/Dockerfile.ci_qemu
+++ b/docker/Dockerfile.ci_qemu
@@ -19,24 +19,18 @@
# tag: v0.62
FROM ubuntu:18.04
-RUN apt-get update --fix-missing
-
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-COPY install/ubuntu1804_install_python_venv.sh /install/ubuntu1804_install_python_venv.sh
-RUN bash /install/ubuntu1804_install_python_venv.sh
-ENV PATH=/opt/tvm-venv/bin:/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
-
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
@@ -77,6 +71,7 @@ COPY install/ubuntu_init_zephyr_project.sh /install/ubuntu_init_zephyr_project.s
COPY install/ubuntu_install_zephyr_sdk.sh /install/ubuntu_install_zephyr_sdk.sh
RUN bash /install/ubuntu_install_zephyr.sh
ENV ZEPHYR_BASE=/opt/zephyrproject/zephyr
+ENV PATH=/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH
# FreeRTOS deps
COPY install/ubuntu_install_freertos.sh /install/ubuntu_install_freertos.sh
diff --git a/docker/Dockerfile.ci_wasm b/docker/Dockerfile.ci_wasm
index 1c7d3eb59b..541c52f5ed 100644
--- a/docker/Dockerfile.ci_wasm
+++ b/docker/Dockerfile.ci_wasm
@@ -16,20 +16,18 @@
# under the License.
FROM ubuntu:18.04
-RUN apt-get update --fix-missing
-
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh
+COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt
COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
-
+COPY python/build/pyproject.toml /install/python/pyproject.toml
+COPY python/build/poetry.lock /install/python/poetry.lock
COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
RUN bash /install/ubuntu_install_python_package.sh
diff --git a/docker/install/ubuntu1804_install_python_venv.sh b/docker/install/ubuntu1804_install_python_venv.sh
deleted file mode 100755
index 5dc5efea76..0000000000
--- a/docker/install/ubuntu1804_install_python_venv.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-# install python and pip, don't modify this, modify install_python_package.sh
-apt-get update
-apt-get install -y software-properties-common
-apt-get install -y python3.7-dev python3-setuptools python3.7-venv
-
-python3 -mvenv /opt/tvm-venv
-
-# Pin pip and setuptools versions
-/opt/tvm-venv/bin/pip3 install pip==19.3.1 setuptools==58.4.0
diff --git a/docker/install/ubuntu2004_install_python.sh b/docker/install/ubuntu2004_install_python.sh
index 5b87a74061..31f803a90f 100755
--- a/docker/install/ubuntu2004_install_python.sh
+++ b/docker/install/ubuntu2004_install_python.sh
@@ -34,12 +34,34 @@ apt-get install -y software-properties-common
apt-get install -y python3.8 python3.8-dev python3-pip
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
-# Pin pip and setuptools versions
-# Hashes generated via:
-# $ pip download <package>==<version>
-# $ pip hash --algorithm sha256 <package>.whl
-cat <<EOF > base-requirements.txt
-pip==22.0.4 --hash=sha256:c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b
-setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036
+function download_hash() {
+ cat >/tmp/hash-bootstrap-packages.py <<EOF
+import os
+import os.path
+import subprocess
+import pkginfo
+
+for f in sorted(os.scandir("."), key=lambda x: x.name):
+ if not f.is_file():
+ continue
+ p = pkginfo.get_metadata(f.name)
+ if not p:
+ continue
+ print(f"{p.name}=={p.version} {subprocess.check_output(['pip3', 'hash', '-a', 'sha256', p.filename], encoding='utf-8').split()[1]} # {f.name}")
EOF
-pip3 install -r base-requirements.txt
+ mkdir packages && cd packages
+ pip3 install -U "$@"
+ pip3 download pip poetry setuptools
+ python3 /tmp/hash-bootstrap-packages.py
+ exit 2 # make docker build stop
+}
+
+# Install bootstrap packages. You can update these with the following procedure:
+# 1. Uncomment the line below, then attempt torebuild the base images (it will fail).
+# 2. New hashes should be printed in the terminal log from each docker build. Copy these hashes into the
+# the arch-appropriate file in docker/python/bootstrap-requirements/
+# download_hash pip setuptools pkginfo
+
+pip3 install -U pip -c /install/python/bootstrap-requirements.txt # Update pip to match version used to produce base-requirements.txt
+pip3 config set global.no-cache-dir false
+pip3 install -r /install/python/bootstrap-requirements.txt -c /install/python/bootstrap-requirements.txt
diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh
index ab71eab54a..6867993c58 100755
--- a/docker/install/ubuntu_install_caffe.sh
+++ b/docker/install/ubuntu_install_caffe.sh
@@ -25,9 +25,6 @@ apt-get install -y --no-install-recommends protobuf-compiler \
libprotobuf-dev libhdf5-serial-dev libopenblas-dev libgflags-dev libgoogle-glog-dev
-# install python packages
-pip install "numpy" "protobuf" "scikit-image" "six"
-
# Build the Caffe and the python wrapper
echo "Downloading Caffe"
CAFFE_HOME="/opt/caffe"
diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh
index 5593d61ea5..07de29b921 100755
--- a/docker/install/ubuntu_install_core.sh
+++ b/docker/install/ubuntu_install_core.sh
@@ -21,7 +21,7 @@ set -u
set -o pipefail
# install libraries for building c++ core on ubuntu
-apt-get update && apt-get install -y --no-install-recommends \
+apt-get update --fix-missing && apt-get install -y --no-install-recommends \
apt-transport-https \
ca-certificates \
cmake \
diff --git a/docker/install/ubuntu_install_coreml.sh b/docker/install/ubuntu_install_coreml.sh
deleted file mode 100755
index cbdc87666b..0000000000
--- a/docker/install/ubuntu_install_coreml.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-pip3 install coremltools
diff --git a/docker/install/ubuntu_install_darknet.sh b/docker/install/ubuntu_install_darknet.sh
deleted file mode 100755
index 8020899f8b..0000000000
--- a/docker/install/ubuntu_install_darknet.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-#install the necessary dependancies, cffi, opencv
-wget -q 'https://github.com/siju-samuel/darknet/blob/master/lib/libdarknet.so?raw=true' -O libdarknet.so
-debian_version=`cat /etc/debian_version`
-
-pip3 install \
- cffi \
- opencv-python
diff --git a/docker/install/ubuntu_install_mxnet.sh b/docker/install/ubuntu_install_mxnet.sh
deleted file mode 100755
index aa04d4c191..0000000000
--- a/docker/install/ubuntu_install_mxnet.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-pip3 install mxnet==1.6.0
diff --git a/docker/install/ubuntu_install_oneflow.sh b/docker/install/ubuntu_install_oneflow.sh
deleted file mode 100755
index 3eb6b7d89b..0000000000
--- a/docker/install/ubuntu_install_oneflow.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-pip3 install flowvision==0.1.0
-
-python3 -m pip install -f https://release.oneflow.info oneflow==0.7.0+cpu
diff --git a/docker/install/ubuntu_install_onnx.sh b/docker/install/ubuntu_install_onnx.sh
deleted file mode 100755
index 6a41a55740..0000000000
--- a/docker/install/ubuntu_install_onnx.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-# We need to fix the onnx version because changing versions tends to break tests
-# TODO(mbrookhart): periodically update
-
-# onnx 1.9 removed onnx optimizer from the main repo (see
-# https://github.com/onnx/onnx/pull/2834). When updating the CI image
-# to onnx>=1.9, onnxoptimizer should also be installed.
-pip3 install \
- onnx==1.10.2 \
- onnxruntime==1.9.0 \
- onnxoptimizer==0.2.6
-
-# torch depends on a number of other packages, but unhelpfully, does
-# not expose that in the wheel!!!
-pip3 install future
-
-pip3 install \
- torch==1.11.0 \
- torchvision==0.12.0 \
- --extra-index-url https://download.pytorch.org/whl/cpu
diff --git a/docker/install/ubuntu_install_paddle.sh b/docker/install/ubuntu_install_paddle.sh
deleted file mode 100755
index c7f9d43a3b..0000000000
--- a/docker/install/ubuntu_install_paddle.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-pip install paddlepaddle==2.1.3
diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh
index 0353814efc..68ed8ea661 100755
--- a/docker/install/ubuntu_install_python_package.sh
+++ b/docker/install/ubuntu_install_python_package.sh
@@ -16,31 +16,13 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-set -u
-set -o pipefail
+set -euo pipefail
+set -x
-# install libraries for python package on ubuntu
-pip3 install --upgrade \
- attrs \
- cloudpickle \
- cython \
- decorator \
- mypy \
- numpy~=1.19.5 \
- orderedset \
- packaging \
- Pillow==9.1.0 \
- psutil \
- pytest \
- tlcpack-sphinx-addon==0.2.1 \
- pytest-profiling \
- pytest-xdist \
- requests \
- scipy \
- Jinja2 \
- synr==0.6.0 \
- junitparser==2.4.2 \
- six \
- tornado \
- pytest-lazy-fixture
+cd $(dirname $0)/python
+poetry config cache-dir /tmp/poetry-cache
+poetry config virtualenvs.path /virtualenv
+
+poetry install --no-root "$@"
+VENV_ROOT=$(ls -d1 /virtualenv/apache-tvm-*-py3.7)
+ln -s "${VENV_ROOT}" /virtualenv/apache-tvm-py3.7
diff --git a/docker/install/ubuntu_install_redis.sh b/docker/install/ubuntu_install_redis.sh
index 8678c20501..615415f557 100755
--- a/docker/install/ubuntu_install_redis.sh
+++ b/docker/install/ubuntu_install_redis.sh
@@ -21,5 +21,3 @@ set -u
set -o pipefail
apt-get update && apt-get install -y redis-server
-pip3 install \
- xgboost==1.4.2
diff --git a/docker/install/ubuntu_install_sphinx.sh b/docker/install/ubuntu_install_sphinx.sh
deleted file mode 100755
index 12ca25b22b..0000000000
--- a/docker/install/ubuntu_install_sphinx.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-# NOTE: install docutils < 0.17 to work around https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
-pip3 install \
- autodocsumm \
- "commonmark>=0.7.3" \
- "docutils>=0.11,<0.17" \
- Image \
- matplotlib \
- sphinx==4.2.0 \
- sphinx_autodoc_annotation \
- sphinx-gallery==0.4.0 \
- sphinx_rtd_theme
diff --git a/docker/install/ubuntu_install_tensorflow.sh b/docker/install/ubuntu_install_tensorflow.sh
deleted file mode 100755
index eaf89ffcf8..0000000000
--- a/docker/install/ubuntu_install_tensorflow.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-pip3 install \
- "h5py==3.1.0" \
- keras==2.6 \
- tensorflow==2.6.2
diff --git a/docker/install/ubuntu_install_tensorflow_aarch64.sh b/docker/install/ubuntu_install_tensorflow_aarch64.sh
index 6acf8b7270..cfb05ac1de 100755
--- a/docker/install/ubuntu_install_tensorflow_aarch64.sh
+++ b/docker/install/ubuntu_install_tensorflow_aarch64.sh
@@ -20,11 +20,3 @@ set -euxo pipefail
# Build dependencies
apt-get install -y --no-install-recommends libhdf5-dev
-
-# We're only using the TensorFlow wheel snapshot here as the
-# h5py wheel tries to use the wrong .so file
-pip3 install \
- "h5py==3.1.0" \
- keras==2.6 \
- tensorflow-aarch64==2.6.2 \
- -f https://snapshots.linaro.org/ldcg/python-cache/tensorflow-aarch64/
diff --git a/docker/install/ubuntu_install_tflite.sh b/docker/install/ubuntu_install_tflite.sh
deleted file mode 100755
index 8a394302fd..0000000000
--- a/docker/install/ubuntu_install_tflite.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-# The tflite version should have matched versions to the tensorflow
-# version installed from pip in ubuntu_install_tensorflow.sh
-TENSORFLOW_VERSION=$(python3 -c "import tensorflow; print(tensorflow.__version__)" 2> /dev/null)
-
-# Download, build and install flatbuffers
-git clone --branch=v1.12.0 --depth=1 --recursive https://github.com/google/flatbuffers.git
-cd flatbuffers
-cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release
-make install -j8
-cd ..
-
-# Install flatbuffers python packages.
-pip3 install flatbuffers
-
-# Build the TFLite static library, necessary for building with TFLite ON.
-# The library is built at:
-# tensorflow/tensorflow/lite/tools/make/gen/*/lib/libtensorflow-lite.a.
-git clone https://github.com/tensorflow/tensorflow --branch=v${TENSORFLOW_VERSION} --depth 1
-./tensorflow/tensorflow/lite/tools/make/download_dependencies.sh
-./tensorflow/tensorflow/lite/tools/make/build_lib.sh
-
-# Setup tflite from schema
-mkdir tflite
-cp tensorflow/tensorflow/lite/schema/schema.fbs tflite
-cd tflite
-flatc --python schema.fbs
-
-cat <<EOM >setup.py
-import setuptools
-
-setuptools.setup(
- name="tflite",
- version="${TENSORFLOW_VERSION}",
- author="google",
- author_email="google@google.com",
- description="TFLite",
- long_description="TFLite",
- long_description_content_type="text/markdown",
- url="https://www.tensorflow.org/lite",
- packages=setuptools.find_packages(),
- classifiers=[
- "Programming Language :: Python :: 2",
- "License :: OSI Approved :: MIT License",
- "Operating System :: OS Independent",
- ],
-)
-EOM
-
-cat <<EOM >__init__.py
-name = "tflite"
-EOM
-
-# Install tflite over python3
-python3 setup.py install
-
-cd ..
-rm -rf tflite
diff --git a/docker/install/ubuntu_install_vela.sh b/docker/install/ubuntu_install_vela.sh
deleted file mode 100755
index c72d118233..0000000000
--- a/docker/install/ubuntu_install_vela.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-pip3 install ethos-u-vela==3.2.0