You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ok...@apache.org on 2023/04/03 14:06:37 UTC
[madlib] 08/08: Update jenkins to use PG13
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch madlib2-master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 03fba10fc0d115afcf25c00e1b97ef747676396a
Author: Orhan Kislal <ok...@apache.org>
AuthorDate: Wed Mar 29 23:01:58 2023 +0300
Update jenkins to use PG13
The current jenkins build is failing because of a server crash.
Added a FIXME in the build script to look at this problem and find an
efficient way to debug in a different JIRA.
---
src/bin/madpack | 2 +-
tool/docker/base/Dockerfile_postgres_13_Jenkins | 40 +++++++++++++++++++++++++
tool/jenkins/jenkins_build.sh | 21 +++++++------
3 files changed, 53 insertions(+), 10 deletions(-)
diff --git a/src/bin/madpack b/src/bin/madpack
index 0507fbc2..66fea155 100755
--- a/src/bin/madpack
+++ b/src/bin/madpack
@@ -13,7 +13,7 @@
# 2. Pass all arguments to ../madpack/madpack.py
PYTHON_PREFIX="python"
-PYTHON_VERSIONS="2.7 2.6"
+PYTHON_VERSIONS="2.7 2.6 3"
# create absolute path to madpack.py
pushd `dirname $0` > /dev/null
diff --git a/tool/docker/base/Dockerfile_postgres_13_Jenkins b/tool/docker/base/Dockerfile_postgres_13_Jenkins
new file mode 100644
index 00000000..59ecd695
--- /dev/null
+++ b/tool/docker/base/Dockerfile_postgres_13_Jenkins
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM postgres:13
+
+### Get postgres specific add-ons
+RUN apt-get update && apt-get install -y wget \
+ build-essential \
+ postgresql-server-dev-13 \
+ postgresql-plpython3-13 \
+ openssl \
+ libssl-dev \
+ libboost-all-dev \
+ m4 \
+ rpm \
+ python3-pip \
+ python3-dev \
+ build-essential \
+ cmake \
+ libspatialindex-dev
+
+RUN python3 -m pip install tensorflow dill rtree xgboost
+
+## To build an image from this docker file, from madlib folder, run:
+# docker build -t madlib/postgres_13:jenkins -f tool/docker/base/Dockerfile_postgres_13_Jenkins .
diff --git a/tool/jenkins/jenkins_build.sh b/tool/jenkins/jenkins_build.sh
index 904a102c..f1df043f 100755
--- a/tool/jenkins/jenkins_build.sh
+++ b/tool/jenkins/jenkins_build.sh
@@ -20,6 +20,9 @@
workdir=`pwd`
user_name=`whoami`
+# FIXME: Jenkins build is not working, even though PG13 compiles on local Mac.
+# Need to find an efficient way to debug since it has a server crash.
+
echo "======================================================================"
echo "Build user: $user_name"
echo "Work directory: $workdir"
@@ -43,14 +46,14 @@ docker rm madlib
echo "Creating docker container"
# Pull down the base docker images
-echo "docker pull madlib/postgres_11:jenkins"
-docker pull madlib/postgres_11:jenkins
+echo "docker pull madlib/postgres_13:jenkins"
+docker pull madlib/postgres_13:jenkins
# Launch docker container with volume mounted from workdir
echo "-------------------------------"
cat <<EOF
-docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_11:jenkins | tee logs/docker_setup.log
+docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_13:jenkins | tee logs/docker_setup.log
EOF
-docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_11:jenkins | tee logs/docker_setup.log
+docker run -d -e POSTGRES_PASSWORD=postgres --name madlib -v "${workdir}":/madlib madlib/postgres_13:jenkins | tee logs/docker_setup.log
echo "-------------------------------"
## This sleep is required since it takes a couple of seconds for the docker
@@ -60,9 +63,9 @@ sleep 15
echo "---------- Install pip, and mock -----------"
# cmake, make, make install, and make package
cat <<EOF
-docker exec madlib bash -c 'apt-get update; apt-get install -y python-pip; pip install mock pandas numpy xgboost scikit-learn' | tee $workdir/logs/madlib_compile.log
+docker exec madlib bash -c 'apt-get update; apt-get install -y python3-pip; pip install mock pandas numpy xgboost scikit-learn yaml' | tee $workdir/logs/madlib_compile.log
EOF
-docker exec madlib bash -c 'apt-get update; apt-get install -y python-pip; pip install mock pandas numpy xgboost scikit-learn' | tee $workdir/logs/madlib_compile.log
+docker exec madlib bash -c 'apt-get update; apt-get install -y python3-pip; pip install mock pandas numpy xgboost scikit-learn yaml' | tee $workdir/logs/madlib_compile.log
echo "---------- Building package -----------"
# cmake, make, make install, and make package
@@ -86,7 +89,7 @@ EOF
docker exec madlib bash -c 'mkdir -p /tmp'
docker exec madlib bash -c 'rm -rf /build/src/ports/postgres/modules/deep_learning/test'
-docker exec madlib bash -c 'rm -rf /build/src/ports/postgres/11/modules/deep_learning/test'
+docker exec madlib bash -c 'rm -rf /build/src/ports/postgres/13/modules/deep_learning/test'
# Run dev check
docker exec madlib bash -c '/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp dev-check' | tee $workdir/logs/madlib_dev_check.log
# Run unit tests, and append output to dev_check's log file
@@ -108,6 +111,6 @@ echo "-------------------------------"
# convert dev-check test results to junit format for reporting
cat <<EOF
-python tool/jenkins/junit_export.py $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml
+python3 tool/jenkins/junit_export.py $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml
EOF
-python tool/jenkins/junit_export.py $workdir $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml
+python3 tool/jenkins/junit_export.py $workdir $workdir/logs/madlib_dev_check.log $workdir/logs/madlib_dev_check.xml