You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by jb...@apache.org on 2018/08/01 21:24:13 UTC
[geode] branch develop updated: Run dockerized tests as geode user.
This is an automated email from the ASF dual-hosted git repository.
jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new ce9cd54 Run dockerized tests as geode user.
ce9cd54 is described below
commit ce9cd5411731d6baf14da4a071095d8314004151
Author: Sean Goller <sg...@pivotal.io>
AuthorDate: Wed Aug 1 14:19:32 2018 -0700
Run dockerized tests as geode user.
* Create new optimized docker image for gradle to use.
* Add geode user with common UID to docker image and google image
* Add reaper pipeline to cleanup script
* Run combineReports before rsyncing down.
Signed-off-by: Dick Cavender <dc...@pivotal.io>
---
ci/docker/Dockerfile | 1 -
ci/images/google-geode-builder/build_image.sh | 2 +-
ci/images/google-geode-builder/scripts/setup.sh | 3 +-
ci/{docker => images/test-container}/Dockerfile | 44 +++----------------------
ci/pipelines/clean_fork_pipelines.sh | 4 +++
ci/pipelines/images/images.yml | 35 ++++++++++++++++++--
ci/scripts/execute_tests.sh | 2 +-
ci/scripts/rsync_code_down.sh | 2 ++
8 files changed, 46 insertions(+), 47 deletions(-)
diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile
index 7e9067b..c7bf95f 100644
--- a/ci/docker/Dockerfile
+++ b/ci/docker/Dockerfile
@@ -18,7 +18,6 @@ ENTRYPOINT []
ARG CHROME_DRIVER_VERSION=2.35
ENV SPRUCE_VERSION 1.17.0
-ENV GRADLE_USER_HOME /usr/local/maven_files
WORKDIR /tmp/work
COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer
diff --git a/ci/images/google-geode-builder/build_image.sh b/ci/images/google-geode-builder/build_image.sh
index 4a369a8..885392f 100755
--- a/ci/images/google-geode-builder/build_image.sh
+++ b/ci/images/google-geode-builder/build_image.sh
@@ -29,7 +29,7 @@ pushd ${SCRIPTDIR}
GEODE_BRANCH=${GEODE_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}
SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
IMAGE_FAMILY_PREFIX=""
-GEODE_DOCKER_IMAGE=${GEODE_DOCKER_IMAGE:-"gcr.io/apachegeode-ci/apachegeode-build-concourse"}
+GEODE_DOCKER_IMAGE=${GEODE_DOCKER_IMAGE:-"gcr.io/apachegeode-ci/test-container"}
if [[ -z "${GEODE_FORK}" ]]; then
echo "GEODE_FORK environment variable must be set for this script to work."
exit 1
diff --git a/ci/images/google-geode-builder/scripts/setup.sh b/ci/images/google-geode-builder/scripts/setup.sh
index 19f66e4..8aa1ad1 100755
--- a/ci/images/google-geode-builder/scripts/setup.sh
+++ b/ci/images/google-geode-builder/scripts/setup.sh
@@ -20,6 +20,7 @@ set -e
export CLOUD_SDK_VERSION=209.0.0
export CHROME_DRIVER_VERSION=2.35
export LOCAL_USER=geode
+export LOCAL_UID=93043
apt-get update
apt-get install -y --no-install-recommends \
@@ -71,7 +72,7 @@ rm /tmp/chromedriver_linux64.zip
mv /opt/selenium/chromedriver /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION}
chmod 755 /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION}
ln -fs /opt/selenium/chromedriver-${CHROME_DRIVER_VERSION} /usr/bin/chromedriver
-adduser --disabled-password --gecos "" ${LOCAL_USER}
+adduser --disabled-password --gecos "" --uid ${LOCAL_UID} ${LOCAL_USER}
usermod -G docker,google-sudoers -a ${LOCAL_USER}
echo "export PATH=/google-cloud-sdk/bin:${PATH}" > /etc/profile.d/google_sdk_path.sh
diff --git a/ci/docker/Dockerfile b/ci/images/test-container/Dockerfile
similarity index 50%
copy from ci/docker/Dockerfile
copy to ci/images/test-container/Dockerfile
index 7e9067b..a8a6d86 100644
--- a/ci/docker/Dockerfile
+++ b/ci/images/test-container/Dockerfile
@@ -17,55 +17,19 @@ FROM openjdk:8
ENTRYPOINT []
ARG CHROME_DRIVER_VERSION=2.35
-ENV SPRUCE_VERSION 1.17.0
-ENV GRADLE_USER_HOME /usr/local/maven_files
WORKDIR /tmp/work
-COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer
-
-ADD https://github.com/geofffranks/spruce/releases/download/v${SPRUCE_VERSION}/spruce-linux-amd64 /usr/local/bin/spruce
-ADD https://github.com/krallin/tini/releases/download/v0.14.0/tini-static-amd64 /usr/local/bin/tini
-ADD tini-wrapper.go .
-ADD ./initdocker /usr/local/bin/initdocker
-ADD cache_dependencies.sh cache_dependencies.sh
-
-RUN chmod +x /usr/local/bin/tini \
- && chmod +x /usr/local/bin/spruce \
- && chmod +x /usr/local/bin/initdocker \
- && chmod +x cache_dependencies.sh \
- && apt-get update \
+RUN apt-get update \
&& apt-get install -y --no-install-recommends \
- apt-transport-https \
- lsb-release \
+ apt-transport-https \
+ lsb-release \
&& echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
- && echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
- && echo "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
&& curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
- && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
- && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
&& apt-get update \
- && apt-get purge lxc-docker \
&& apt-get install -y --no-install-recommends \
aptitude \
ca-certificates \
- cgroupfs-mount \
- docker-compose \
- docker-ce \
- golang \
google-chrome-stable \
- google-cloud-sdk \
- htop \
- jq \
- python3 \
- python3-pip \
- unzip \
- vim \
- && gcloud config set core/disable_usage_reporting true \
- && gcloud config set component_manager/disable_update_check true \
- && gcloud config set metrics/environment github_docker_image \
- && go build -o /usr/local/bin/tini-wrapper ./tini-wrapper.go \
- && curl -Lo /usr/local/bin/dunit-progress https://github.com/jdeppe-pivotal/progress-util/releases/download/0.2/progress.linux \
- && chmod +x /usr/local/bin/dunit-progress \
&& wget --no-verbose -O /tmp/chromedriver_linux64.zip https://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip \
&& rm -rf /opt/selenium/chromedriver \
&& unzip /tmp/chromedriver_linux64.zip -d /opt/selenium \
@@ -73,7 +37,7 @@ RUN chmod +x /usr/local/bin/tini \
&& mv /opt/selenium/chromedriver /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION \
&& chmod 755 /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION \
&& ln -fs /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION /usr/bin/chromedriver \
- && ./cache_dependencies.sh \
+ && useradd --shell /bin/bash -u 93043 -o -c "" -m geode \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/work
diff --git a/ci/pipelines/clean_fork_pipelines.sh b/ci/pipelines/clean_fork_pipelines.sh
index 4d6c03b..903dcf1 100755
--- a/ci/pipelines/clean_fork_pipelines.sh
+++ b/ci/pipelines/clean_fork_pipelines.sh
@@ -38,6 +38,10 @@ echo "Deleting images pipeline if it exists..."
IMAGES_PIPELINE="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-images"
fly -t ${TARGET} destroy-pipeline --non-interactive -p ${IMAGES_PIPELINE}
+echo "Deleting reaper pipeline if it exists..."
+REAPER_PIPELINE="${GEOD_FORK}-${SANITIZED_GEODE_BRANCH}-reaper"
+fly -t ${TARGET} destroy-pipeline --non-interactive -p ${REAPER_PIPELINE}
+
echo "Deleting build pipeline if it exists..."
BUILD_PIPELINE="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}"
fly -t ${TARGET} destroy-pipeline --non-interactive -p ${BUILD_PIPELINE}
diff --git a/ci/pipelines/images/images.yml b/ci/pipelines/images/images.yml
index 1b1914e..0864038 100644
--- a/ci/pipelines/images/images.yml
+++ b/ci/pipelines/images/images.yml
@@ -76,6 +76,21 @@ resources:
password: ((!docker-password))
repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
+- name: test-container-dockerfile
+ type: git
+ source:
+ uri: https://github.com/((!geode-fork))/geode.git
+ branch: ((!geode-build-branch))
+ paths:
+ - ci/images/test-container/*
+
+- name: test-container-docker-image
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/((!docker-image-prefix))test-container
+
- name: concourse-metadata-resource-dockerfile
type: git
source:
@@ -127,8 +142,8 @@ jobs:
serial: true
plan:
- aggregate:
- - get: apachegeode-build-concourse-docker-image
- passed: [build-apachegeode-build-concourse-docker-image]
+ - get: test-container-docker-image
+ passed: [build-test-container-docker-image]
trigger: true
- get: google-geode-builder
trigger: true
@@ -144,7 +159,7 @@ jobs:
params:
GEODE_FORK: ((!geode-fork))
GEODE_BRANCH: ((!geode-build-branch))
- GEODE_DOCKER_IMAGE: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
+ GEODE_DOCKER_IMAGE: gcr.io/apachegeode-ci/((!docker-image-prefix))test-container
run:
path: google-geode-builder/ci/images/google-geode-builder/build_image.sh
@@ -162,6 +177,20 @@ jobs:
build: alpine-tools-dockerfile/ci/images/alpine-tools
tag_as_latest: true
+- name: build-test-container-docker-image
+ public: ((!public-pipelines))
+ serial: true
+ plan:
+ - aggregate:
+ - get: alpine37-docker-image
+ trigger: true
+ - get: test-container-dockerfile
+ trigger: true
+ - put: test-container-docker-image
+ params:
+ build: test-container-dockerfile/ci/images/test-container
+ tag_as_latest: true
+
- name: build-concourse-metadata-resource-docker-image
public: ((!public-pipelines))
serial: true
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index 6c203c8..650d2e1 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -54,7 +54,7 @@ scp -i ${SSHKEY_FILE} ${SCRIPTDIR}/capture-call-stacks.sh geode@${INSTANCE_IP_AD
if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then
- PARALLEL_DUNIT="-PparallelDunit"
+ PARALLEL_DUNIT="-PparallelDunit -PdunitDockerUser=geode"
if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then
DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}"
fi
diff --git a/ci/scripts/rsync_code_down.sh b/ci/scripts/rsync_code_down.sh
index 47659f7..b203dcc 100755
--- a/ci/scripts/rsync_code_down.sh
+++ b/ci/scripts/rsync_code_down.sh
@@ -40,5 +40,7 @@ echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
OUTPUT_DIR=${BASE_DIR}/geode-results
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "cd geode && ./gradlew combineReports"
+
time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:. ${OUTPUT_DIR}/.
set +x