You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by dj...@apache.org on 2025/12/18 08:21:39 UTC
(cloudberry-pxf) 34/37: added specs for gpdb7-rhel8-test-pxf docker image (#867)
This is an automated email from the ASF dual-hosted git repository.
djwang pushed a commit to branch merge-with-upstream
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 26c0920e3513c12a2caeb1255c5ae9a54a7d9fbe
Author: Alexander Denissov <de...@users.noreply.github.com>
AuthorDate: Thu Sep 15 17:00:27 2022 -0700
added specs for gpdb7-rhel8-test-pxf docker image (#867)
---
concourse/docker/pxf-dev-base/README.md | 19 ++-
concourse/docker/pxf-dev-base/cloudbuild.yaml | 33 ++++-
.../docker/pxf-dev-base/gpdb7/rhel8/Dockerfile | 25 ++--
concourse/pipelines/cloudbuild_pipeline.yml | 148 +++++++++++++++++++--
4 files changed, 197 insertions(+), 28 deletions(-)
diff --git a/concourse/docker/pxf-dev-base/README.md b/concourse/docker/pxf-dev-base/README.md
index 69418887..b905fb1c 100644
--- a/concourse/docker/pxf-dev-base/README.md
+++ b/concourse/docker/pxf-dev-base/README.md
@@ -132,7 +132,9 @@ following command to build the image:
### Docker gpdb7-rocky8-test-pxf-image image
-Build this image for Greenplum 7 running on Rocky 8. Run the following
+TODO: ===> remove this if Greenplum 7 will not be supported on Centos7 <===
+
+Build this image for Greenplum 7 running on CentOS 7. Run the following
command to build the image:
pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/
@@ -145,6 +147,21 @@ command to build the image:
.
popd
+### Docker gpdb7-rhel8-test-pxf-image image
+
+Build this image for Greenplum 7 running on Rhel 8. Run the following
+command to build the image:
+
+ pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/
+ docker build \
+ --build-arg=BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest \
+ --build-arg=GINKGO_VERSION=${GINKGO_VERSION} \
+ --tag=gpdb7-rhel8-test-pxf \
+ -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile \
+ .
+ popd
+
+### Docker gpdb7-ubuntu18.04-test-pxf-image image
### Docker gpdb7-rocky9-test-pxf-image image
diff --git a/concourse/docker/pxf-dev-base/cloudbuild.yaml b/concourse/docker/pxf-dev-base/cloudbuild.yaml
index 1b999d0c..d7fd8d10 100644
--- a/concourse/docker/pxf-dev-base/cloudbuild.yaml
+++ b/concourse/docker/pxf-dev-base/cloudbuild.yaml
@@ -224,7 +224,33 @@ steps:
- apache-maven-binaries
- gpdb7-rocky8-test-pxf-image-cache
- # Greenplum 7 Rocky 9 Image
+# Greenplum 7 Rhel 8 Image
+- name: 'gcr.io/cloud-builders/docker'
+ id: gpdb7-rhel8-test-pxf-image-cache
+ entrypoint: 'bash'
+ args:
+ - '-c'
+ - |
+ mkdir -p /workspace/build
+ docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest || exit 0
+ waitFor: ['-']
+
+- name: 'gcr.io/cloud-builders/docker'
+ id: gpdb7-rhel8-test-pxf-image
+ args:
+ - 'build'
+ - '--build-arg=BASE_IMAGE=${_PRIVATE_BASE_IMAGE_REPOSITORY}/gpdb7-rhel8-test:latest'
+ - '--build-arg=GINKGO_VERSION=${_GINKGO_VERSION}'
+ - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA'
+ - '--cache-from'
+ - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest'
+ - '-f'
+ - 'concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile'
+ - '/workspace/build/'
+ waitFor:
+ - gpdb7-rhel8-test-pxf-image-cache
+
+# Greenplum 7 Ubuntu 18.04 Image
- name: 'gcr.io/cloud-builders/docker'
id: gpdb7-rocky9-test-pxf-image-cache
entrypoint: 'bash'
@@ -265,5 +291,6 @@ images:
- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky9-test-pxf:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:$COMMIT_SHA'
-- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:$COMMIT_SHA'
-- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky9-test-pxf:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA'
+- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA'
diff --git a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile
index 39ccd433..6217f14c 100644
--- a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile
+++ b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile
@@ -2,16 +2,10 @@ ARG BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest
FROM ${BASE_IMAGE}
-ARG GO_VERSION
-ARG GO_SHA256SUM
-
-ADD apache-maven.tar.gz /usr/share
+ARG GINKGO_VERSION
# install Go utilities
-RUN mkdir -p /tmp/pxf_src/ && cd /tmp \
- && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \
- && echo "${GO_SHA256SUM} /tmp/go.tgz" | sha256sum --check \
- && rm -rf /usr/local/go && tar -C /usr/local -xzf go.tgz && rm go.tgz
+RUN GOPATH=/opt/go /usr/local/go/bin/go install github.com/onsi/ginkgo/ginkgo@v${GINKGO_VERSION}
# add minio software
RUN useradd -s /sbin/nologin -d /opt/minio minio \
@@ -22,8 +16,17 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \
&& chmod +x /opt/minio/bin/minio \
&& chown -R minio:minio /opt/minio
+ARG MAVEN_VERSION=3.6.3
+ARG USER_HOME_DIR="/root"
+ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+
# install dependencies that are missing on the base images
-RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn
+RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
+ && mkdir -p /usr/share/maven \
+ && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \
+ && rm -f /tmp/apache-maven.tar.gz \
+ && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \
+ && /usr/bin/pip2 install paramiko --no-cache-dir
# create rsa key for the root user
RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \
@@ -58,11 +61,11 @@ RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \
&& echo >> ~gpadmin/.pxfrc 'export PGPORT=5432' \
&& echo >> ~gpadmin/.pxfrc 'export GOPATH=/opt/go' \
&& echo >> ~gpadmin/.pxfrc 'export GPHOME=$(find /usr/local/ -name greenplum-db* -type d | head -n1)' \
+ && echo >> ~gpadmin/.pxfrc 'export LD_LIBRARY_PATH=${GPHOME}/lib:${LD_LIBRARY_PATH-}' \
&& echo >> ~gpadmin/.pxfrc 'export GPHD_ROOT=/singlecluster' \
&& echo >> ~gpadmin/.pxfrc 'export PXF_HOME=/usr/local/pxf-gp7' \
&& echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk' \
- && echo >> ~gpadmin/.pxfrc 'export PG_CONFIG=${GPHOME}/bin/pg_config' \
- && echo >> ~gpadmin/.pxfrc 'export PATH=${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$PATH' \
+ && echo >> ~gpadmin/.pxfrc 'export PATH=${GPHOME}/bin:${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$JAVA_HOME/bin:$PATH' \
&& ln -s ~gpadmin/.pxfrc ~root \
&& echo >> ~gpadmin/.bashrc 'source ~/.pxfrc' \
&& chown -R gpadmin:gpadmin ~gpadmin
diff --git a/concourse/pipelines/cloudbuild_pipeline.yml b/concourse/pipelines/cloudbuild_pipeline.yml
index d82a92bb..b8709082 100644
--- a/concourse/pipelines/cloudbuild_pipeline.yml
+++ b/concourse/pipelines/cloudbuild_pipeline.yml
@@ -92,16 +92,138 @@ jobs:
- name: pxf_src
run:
path: pxf_src/concourse/scripts/check_docker_images_and_tag.bash
- - task: trigger_builds
- image: gcloud
- params:
- GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key))
- GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id))
- GOOGLE_ZONE: ((ud/pxf/common/google-zone))
- TRIGGER_LIST: "pxf-dev-server"
- config:
- platform: linux
- inputs:
- - name: pxf_src
- run:
- path: pxf_src/concourse/scripts/trigger_cloudbuild.bash
+
+## ======================================================================
+## RESOURCES
+## ======================================================================
+resources:
+ - name: gcloud
+ type: registry-image
+ icon: docker
+ source:
+ repository: google/cloud-sdk
+ tag: slim
+
+ - name: pxf-dev-base-cloudbuild-trigger
+ type: git
+ icon: git
+ source:
+ branch: ((pxf-git-branch))
+ uri: ((ud/pxf/common/git-remote))
+ paths:
+ - concourse/docker/pxf-dev-base
+
+ - name: dockerfile-gpdb-pxf-mapr
+ type: git
+ icon: git
+ source:
+ branch: ((pxf-git-branch))
+ uri: ((ud/pxf/common/git-remote))
+ paths:
+ - concourse/docker/mapr
+
+ - name: base-images-trigger
+ type: git
+ icon: git
+ source:
+ uri: ((ud/pxf/common/gp-image-baking-repo-uri))
+ branch: ((ud/pxf/common/gp-image-baking-repo-branch))
+ private_key: ((ud/pxf/secrets/gp-image-baking-repo-key))
+ paths:
+ - images/docker/gpdb5/gpdb5-centos7-build-test
+ - images/docker/gpdb6/gpdb6-centos7-test
+ - images/docker/gpdb6/gpdb6-ubuntu18.04-test
+ - images/docker/gpdb6/gpdb6-oel7-test
+ - images/docker/gpdb6/gpdb6-rhel8-test
+ - images/docker/gpdb7/gpdb7-centos7-test
+ - images/docker/gpdb7/gpdb7-rhel8-test
+
+## ======================================================================
+## JOBS
+## ======================================================================
+jobs:
+ - name: Check upstream base images
+ max_in_flight: 1
+ plan:
+ - in_parallel:
+ - get: gcloud
+ - get: pxf_src
+ resource: pxf-dev-base-cloudbuild-trigger
+ - get: base-images-trigger
+ trigger: true
+ - task: trigger_builds
+ image: gcloud
+ params:
+ GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key))
+ GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id))
+ GOOGLE_ZONE: ((ud/pxf/common/google-zone))
+ TRIGGER_LIST: "pxf-dev-base rpmrebuild"
+ config:
+ platform: linux
+ inputs:
+ - name: pxf_src
+ run:
+ path: pxf_src/concourse/scripts/trigger_cloudbuild.bash
+
+ - name: Check pxf-dev-base images
+ max_in_flight: 1
+ plan:
+ - in_parallel:
+ - get: gcloud
+ - get: pxf_src
+ resource: pxf-dev-base-cloudbuild-trigger
+ trigger: true
+ - get: base-images-trigger
+ passed: [Check upstream base images]
+ trigger: true
+ - task: check_docker_images_and_tag
+ timeout: 30m
+ image: gcloud
+ params:
+ GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key))
+ GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id))
+ GOOGLE_ZONE: ((ud/pxf/common/google-zone))
+ IMAGE_LIST: "gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb6-rhel8-test-pxf gpdb6-ubuntu18.04-test-pxf gpdb6-oel7-test-pxf gpdb7-centos7-test-pxf gpdb7-rhel8-test-pxf gpdb7-ubuntu18.04-test-pxf"
+ config:
+ platform: linux
+ inputs:
+ - name: pxf_src
+ run:
+ path: pxf_src/concourse/scripts/check_docker_images_and_tag.bash
+ - task: trigger_builds
+ image: gcloud
+ params:
+ GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key))
+ GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id))
+ GOOGLE_ZONE: ((ud/pxf/common/google-zone))
+ TRIGGER_LIST: "pxf-dev-server pxf-dev-mapr"
+ config:
+ platform: linux
+ inputs:
+ - name: pxf_src
+ run:
+ path: pxf_src/concourse/scripts/trigger_cloudbuild.bash
+
+ - name: Check pxf-dev-mapr images
+ max_in_flight: 1
+ plan:
+ - in_parallel:
+ - get: gcloud
+ - get: pxf_src
+ resource: dockerfile-gpdb-pxf-mapr
+ trigger: true
+ - do: *pxf_dev_mapr_image
+
+ - name: Check pxf-dev-mapr images 2
+ max_in_flight: 1
+ plan:
+ - in_parallel:
+ - get: gcloud
+ - get: pxf_src
+ resource: pxf-dev-base-cloudbuild-trigger
+ passed: [Check pxf-dev-base images]
+ trigger: true
+ - get: base-images-trigger
+ passed: [Check pxf-dev-base images]
+ trigger: true
+ - do: *pxf_dev_mapr_image
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org