You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by dg...@apache.org on 2023/02/28 02:28:19 UTC
[openwhisk-runtime-python] branch master updated: Add Python 3.11 Runtime (#140)
This is an automated email from the ASF dual-hosted git repository.
dgrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/openwhisk-runtime-python.git
The following commit(s) were added to refs/heads/master by this push:
new b24075e Add Python 3.11 Runtime (#140)
b24075e is described below
commit b24075e37d7cf04b00f2a4f36793596393bc436c
Author: Luke-Roy-IBM <83...@users.noreply.github.com>
AuthorDate: Tue Feb 28 03:28:12 2023 +0100
Add Python 3.11 Runtime (#140)
Add Pyton 3.11 Runtime (Version as of commit: 3.11.2)
Using Golang 1.20 Based Proxy
Update Pip modules
use fixed packages for 3.11
---
.github/workflows/ci.yaml | 4 ++
README.md | 3 +-
core/python311Action/Dockerfile | 70 ++++++++++++++++++++++
.../python311Action/build.gradle | 57 +++++++++---------
core/python311Action/requirements.txt | 15 +++++
core/requirements_common.txt | 2 +-
settings.gradle | 1 +
tests/src/test/resources/build.sh | 2 +-
.../runtime/actionContainers/Python311Tests.scala | 33 ++++------
9 files changed, 132 insertions(+), 55 deletions(-)
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b3fc2b0..45c4c66 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -96,6 +96,8 @@ jobs:
./gradlew :core:python39Action:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk -PdockerImageTag=$SHORT_COMMIT
./gradlew :core:python310Action:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk -PdockerImageTag=nightly
./gradlew :core:python310Action:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk -PdockerImageTag=$SHORT_COMMIT
+ ./gradlew :core:python311Action:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk -PdockerImageTag=nightly
+ ./gradlew :core:python311Action:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk -PdockerImageTag=$SHORT_COMMIT
- name: Push Release Images
if: ${{ env.PUSH_RELEASE == 'true' }}
working-directory: runtime
@@ -110,5 +112,7 @@ jobs:
RUNTIME="python39Action"
elif [ ${RUNTIME_VERSION} == "310" ]; then
RUNTIME="python310Action"
+ elif [ ${RUNTIME_VERSION} == "311" ]; then
+ RUNTIME="python311Action"
fi
./gradlew :core:$RUNTIME:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk -PdockerImageTag=$IMAGE_TAG
diff --git a/README.md b/README.md
index 5db1a8a..f329f5f 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,7 @@ The following Python runtime versions (with kind & image labels) are generated b
- Python 3.7 (python:3.7 & openwhisk/action-python-v3.7)
- Python 3.9 (python:3.9 & openwhisk/action-python-v3.9)
- Python 3.10 (python:3.10 & openwhisk/action-python-v3.10)
+- Python 3.11 (python:3.11 & openwhisk/action-python-v3.11)
- Python 3.6 AI (python:3.6 & openwhisk/action-python-v3.6-ai)
This README documents the build, customization and testing of these runtime images.
@@ -184,6 +185,6 @@ If you have an action in the format described before (with a `requirements.txt`)
zip -j -r myaction | docker run -i action-python-v3.7 -compile main > myaction.zip
```
-You may use `v3.10`, `v3.9` or `v3.6-ai` as well according to your Python version needs.
+You may use `v3.11`, `v3.10`, `v3.9` or `v3.6-ai` as well according to your Python version needs.
The resulting action includes a virtualenv already built for you and that is fast to deploy and start as all the dependencies are already resolved. Note that there is a limit on the size of the zip file and this approach will not work for installing large libraries like Pandas or Numpy, instead use the provide "v.3.6-ai" runtime instead which provides these libraries already for you.
diff --git a/core/python311Action/Dockerfile b/core/python311Action/Dockerfile
new file mode 100644
index 0000000..2709f31
--- /dev/null
+++ b/core/python311Action/Dockerfile
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# build go proxy from source
+FROM golang:1.20 AS builder_source
+ARG GO_PROXY_GITHUB_USER=apache
+ARG GO_PROXY_GITHUB_BRANCH=master
+RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \
+ https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\
+ cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \
+ mv proxy /bin/proxy
+
+# or build it from a release
+FROM golang:1.20 AS builder_release
+ARG GO_PROXY_RELEASE_VERSION=1.20@1.21.0
+RUN curl -sL \
+ https://github.com/apache/openwhisk-runtime-go/archive/{$GO_PROXY_RELEASE_VERSION}.tar.gz\
+ | tar xzf -\
+ && cd openwhisk-runtime-go-*/main\
+ && GO111MODULE=on CGO_ENABLED=0 go build -o /bin/proxy
+
+FROM python:3.11-buster
+
+# select the builder to use
+ARG GO_PROXY_BUILD_FROM=release
+
+# install zip
+RUN apt-get update && apt-get install -y zip gcc \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install common modules for python
+COPY requirements_common.txt requirements_common.txt
+COPY requirements.txt requirements.txt
+RUN pip3 install --upgrade pip six wheel &&\
+ pip3 install --no-cache-dir -r requirements.txt
+
+RUN mkdir -p /action
+WORKDIR /
+
+COPY --from=builder_source /bin/proxy /bin/proxy_source
+COPY --from=builder_release /bin/proxy /bin/proxy_release
+RUN mv /bin/proxy_${GO_PROXY_BUILD_FROM} /bin/proxy
+
+ADD bin/compile /bin/compile
+ADD lib/launcher.py /lib/launcher.py
+
+# log initialization errors
+ENV OW_LOG_INIT_ERROR=1
+# the launcher must wait for an ack
+ENV OW_WAIT_FOR_ACK=1
+# using the runtime name to identify the execution environment
+ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.11
+# compiler script
+ENV OW_COMPILER=/bin/compile
+
+ENTRYPOINT ["/bin/proxy"]
diff --git a/settings.gradle b/core/python311Action/build.gradle
similarity index 53%
copy from settings.gradle
copy to core/python311Action/build.gradle
index cc3e542..fda9bbe 100644
--- a/settings.gradle
+++ b/core/python311Action/build.gradle
@@ -15,32 +15,31 @@
* limitations under the License.
*/
-include 'tests'
-
-include 'core:python3Action'
-include 'core:python36AiAction'
-include 'core:python39Action'
-include 'core:python310Action'
-
-rootProject.name = 'runtime-python'
-
-gradle.ext.openwhisk = [
- version: '1.0.1-SNAPSHOT'
-]
-
-gradle.ext.scala = [
- version: '2.12.10',
- depVersion : '2.12',
- compileFlags: ['-feature', '-unchecked', '-deprecation', '-Xfatal-warnings', '-Ywarn-unused-import']
-]
-
-gradle.ext.akka = [version : '2.6.12']
-gradle.ext.akka_http = [version : '10.2.4']
-
-gradle.ext.scalafmt = [
- version: '1.5.0',
- config: new File(rootProject.projectDir, '.scalafmt.conf')
-]
-
-gradle.ext.akka = [version: '2.6.12']
-gradle.ext.akka_http = [version: '10.2.4']
+ext.dockerImageName = 'action-python-v3.11'
+apply from: '../../gradle/docker.gradle'
+
+distDocker.dependsOn 'copyLib'
+distDocker.dependsOn 'copyBin'
+distDocker.dependsOn 'copyReqrCommon'
+distDocker.finalizedBy('cleanup')
+
+task copyLib(type: Copy) {
+ from '../python3Action/lib'
+ into './lib'
+}
+
+task copyBin(type: Copy) {
+ from '../python3Action/bin'
+ into './bin'
+}
+
+task copyReqrCommon(type: Copy) {
+ from '../requirements_common.txt'
+ into './'
+}
+
+task cleanup(type: Delete) {
+ delete 'bin'
+ delete 'lib'
+ delete 'requirements_common.txt'
+}
diff --git a/core/python311Action/requirements.txt b/core/python311Action/requirements.txt
new file mode 100644
index 0000000..cd2ff99
--- /dev/null
+++ b/core/python311Action/requirements.txt
@@ -0,0 +1,15 @@
+# default packages available for action-python-v3.11
+#-r requirements_common.txt
+beautifulsoup4==4.11.2
+httplib2==0.21.0
+kafka_python==2.0.2
+lxml==4.9.2
+python-dateutil==2.8.2
+requests==2.28.2
+scrapy==2.8.0
+simplejson==3.18.3
+virtualenv==20.19.0
+twisted==22.10.0
+netifaces==0.11.0
+# fix issue: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools
+setuptools == 67.4.0
\ No newline at end of file
diff --git a/core/requirements_common.txt b/core/requirements_common.txt
index 88d2c86..6495338 100644
--- a/core/requirements_common.txt
+++ b/core/requirements_common.txt
@@ -9,4 +9,4 @@ scrapy==2.5.0
simplejson==3.17.5
virtualenv==20.8.0
twisted==21.7.0
-netifaces==0.11.0
+netifaces==0.11.0
\ No newline at end of file
diff --git a/settings.gradle b/settings.gradle
index cc3e542..55fd2e4 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -21,6 +21,7 @@ include 'core:python3Action'
include 'core:python36AiAction'
include 'core:python39Action'
include 'core:python310Action'
+include 'core:python311Action'
rootProject.name = 'runtime-python'
diff --git a/tests/src/test/resources/build.sh b/tests/src/test/resources/build.sh
index b3c72db..9ae95c9 100755
--- a/tests/src/test/resources/build.sh
+++ b/tests/src/test/resources/build.sh
@@ -21,7 +21,7 @@ if [ -f ".built" ]; then
exit 0
fi
-for i in v3.7 v3.6-ai v3.9 v3.10
+for i in v3.7 v3.6-ai v3.9 v3.10 v3.11
do echo "*** $i ***"
zip -r -j - python_virtualenv | docker run -i action-python-$i -compile main >python-${i}_virtualenv.zip
cp python-${i}_virtualenv.zip python-${i}_virtualenv_invalid_main.zip
diff --git a/settings.gradle b/tests/src/test/scala/runtime/actionContainers/Python311Tests.scala
similarity index 54%
copy from settings.gradle
copy to tests/src/test/scala/runtime/actionContainers/Python311Tests.scala
index cc3e542..e758872 100644
--- a/settings.gradle
+++ b/tests/src/test/scala/runtime/actionContainers/Python311Tests.scala
@@ -15,32 +15,19 @@
* limitations under the License.
*/
-include 'tests'
+package runtime.actionContainers
-include 'core:python3Action'
-include 'core:python36AiAction'
-include 'core:python39Action'
-include 'core:python310Action'
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
-rootProject.name = 'runtime-python'
+@RunWith(classOf[JUnitRunner])
+class Python311Tests extends Python37Tests {
-gradle.ext.openwhisk = [
- version: '1.0.1-SNAPSHOT'
-]
+ override lazy val imageName = "action-python-v3.11"
-gradle.ext.scala = [
- version: '2.12.10',
- depVersion : '2.12',
- compileFlags: ['-feature', '-unchecked', '-deprecation', '-Xfatal-warnings', '-Ywarn-unused-import']
-]
+ override lazy val zipPrefix = "python-v3.11"
-gradle.ext.akka = [version : '2.6.12']
-gradle.ext.akka_http = [version : '10.2.4']
+ override lazy val errorCodeOnRun = false
-gradle.ext.scalafmt = [
- version: '1.5.0',
- config: new File(rootProject.projectDir, '.scalafmt.conf')
-]
-
-gradle.ext.akka = [version: '2.6.12']
-gradle.ext.akka_http = [version: '10.2.4']
+ override val testNoSource = TestConfig("", hasCodeStub = false)
+}