You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by sa...@apache.org on 2018/08/09 16:35:38 UTC
[geode] branch develop updated: GEODE-5212: Add windows jobs to
develop pipeline (#2284)
This is an automated email from the ASF dual-hosted git repository.
sai_boorlagadda pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new e637894 GEODE-5212: Add windows jobs to develop pipeline (#2284)
e637894 is described below
commit e63789464dbce576fcdd6ac5dfc3a2d390426608
Author: Sai Boorlagadda <sa...@gmail.com>
AuthorDate: Thu Aug 9 09:35:33 2018 -0700
GEODE-5212: Add windows jobs to develop pipeline (#2284)
---
ci/pipelines/geode-build/base.yml | 6 +-
ci/pipelines/geode-build/deploy_pipeline.sh | 11 ++
.../test-stubs/windows/windows-acceptance.yml | 23 ++++
.../test-stubs/windows/windows-distributed.yml | 24 ++++
.../test-stubs/windows/windows-integration.yml | 23 ++++
.../test-stubs/windows/windows-unit.yml | 23 ++++
ci/pipelines/geode-build/windows-test-template.yml | 89 ++++++++++++++
ci/scripts/windows/test-archive.sh | 130 +++++++++++++++++++++
ci/scripts/windows/test-run.sh | 94 +++++++++++++++
9 files changed, 422 insertions(+), 1 deletion(-)
diff --git a/ci/pipelines/geode-build/base.yml b/ci/pipelines/geode-build/base.yml
index 93d41aa..f199e3f 100644
--- a/ci/pipelines/geode-build/base.yml
+++ b/ci/pipelines/geode-build/base.yml
@@ -93,6 +93,10 @@ groups:
- DistributedTest
- AcceptanceTest
- IntegrationTest
+ - WindowsUnitTests
+ - WindowsIntegrationTests
+ - WindowsAcceptanceTests
+ - WindowsGfshDistributedTest
- UpgradeTest
- UpdatePassingRef
@@ -166,4 +170,4 @@ jobs:
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
run:
- path: geode-ci/ci/scripts/update-passing-ref.sh
+ path: geode-ci/ci/scripts/update-passing-ref.sh
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh
index 59c22ab..1be9812 100755
--- a/ci/pipelines/geode-build/deploy_pipeline.sh
+++ b/ci/pipelines/geode-build/deploy_pipeline.sh
@@ -66,6 +66,17 @@ for i in ${SCRIPTDIR}/test-stubs/*.yml; do
${i} > ${TMP_DIR}/${X}
done
+for i in ${SCRIPTDIR}/test-stubs/windows/*.yml; do
+ X=$(basename $i)
+ echo "Merging ${i} into ${TMP_DIR}/${X}"
+ ${SPRUCE} merge --prune metadata \
+ <(echo "metadata:"; \
+ echo " geode-build-branch: ${GEODE_BRANCH}"; \
+ echo " geode-fork: ${GEODE_FORK}") \
+ ${SCRIPTDIR}/windows-test-template.yml \
+ ${i} > ${TMP_DIR}/${X}
+done
+
echo "Spruce branch-name into resources"
${SPRUCE} merge --prune metadata \
${SCRIPTDIR}/base.yml \
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-acceptance.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-acceptance.yml
new file mode 100644
index 0000000..a414f2e
--- /dev/null
+++ b/ci/pipelines/geode-build/test-stubs/windows/windows-acceptance.yml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metadata:
+ job:
+ name: WindowsAcceptanceTest
+ gradle_task: :geode-assembly:acceptanceTest
+ artifact_slug: windows-acceptancetestfiles
+ call_stack_timeout: 1800
+ tags: windows-acceptance-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-distributed.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-distributed.yml
new file mode 100644
index 0000000..d719bbb
--- /dev/null
+++ b/ci/pipelines/geode-build/test-stubs/windows/windows-distributed.yml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metadata:
+ job:
+ name: WindowsGfshDistributedTest
+ gradle_task: distributedTest
+ artifact_slug: windows-gfshdistributedtest
+ test_category: org.apache.geode.test.junit.categories.GfshTest
+ call_stack_timeout: 1800
+ tags: windows-distributed-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-integration.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-integration.yml
new file mode 100644
index 0000000..2c3b785
--- /dev/null
+++ b/ci/pipelines/geode-build/test-stubs/windows/windows-integration.yml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metadata:
+ job:
+ name: WindowsIntegrationTest
+ gradle_task: integrationTest
+ artifact_slug: windows-integrationtestfiles
+ call_stack_timeout: 1800
+ tags: windows-integration-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-unit.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-unit.yml
new file mode 100644
index 0000000..2b704f3
--- /dev/null
+++ b/ci/pipelines/geode-build/test-stubs/windows/windows-unit.yml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+metadata:
+ job:
+ name: WindowsUnitTest
+ gradle_task: test
+ artifact_slug: windows-unittestfiles
+ call_stack_timeout: 1800
+ tags: windows-unit-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/windows-test-template.yml b/ci/pipelines/geode-build/windows-test-template.yml
new file mode 100644
index 0000000..a2f916a
--- /dev/null
+++ b/ci/pipelines/geode-build/windows-test-template.yml
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+jobs:
+- name: (( grab metadata.job.name ))
+ serial: false
+ public: true
+ plan:
+ - get: geode-ci
+ - aggregate:
+ - get: geode
+ passed: [Build]
+ - get: geode-build-version
+ passed: [Build]
+ trigger: true
+ - task: cleanup-java-processes
+ tags: [(( grab metadata.job.tags ))]
+ config:
+ platform: windows
+ run:
+ path: powershell
+ args:
+ - -command
+ - |
+ gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
+ kill -name java -force
+ exit 0
+ - task: execute_tests
+ timeout: 4h
+ tags: [(( grab metadata.job.tags ))]
+ config:
+ inputs:
+ - name: geode-ci
+ - name: geode
+ outputs:
+ - name: built-geode
+ platform: windows
+ params:
+ CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
+ DUNIT_PARALLEL_FORKS: 0
+ MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+ PARALLEL_DUNIT: false
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ JAVA_HOME: "C:\\progra~1\\java\\jdk1.8.0_181"
+ run:
+ path: bash
+ args:
+ - geode-ci/ci/scripts/windows/test-run.sh
+ - (( grab metadata.job.gradle_task ))
+ - (( grab metadata.job.artifact_slug ))
+ - (( grab metadata.job.test_category || "" ))
+ ensure:
+ aggregate:
+ - task: archive_results
+ timeout: 1h
+ tags: [(( grab metadata.job.tags ))]
+ config:
+ inputs:
+ - name: geode-ci
+ - name: geode-build-version
+ - name: built-geode
+ platform: windows
+ params:
+ MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ PUBLIC_BUCKET: ((!public-bucket))
+ run:
+ path: bash
+ args:
+ - geode-ci/ci/scripts/windows/test-archive.sh
+ - (( grab metadata.job.gradle_task ))
+ - (( grab metadata.job.artifact_slug ))
+
diff --git a/ci/scripts/windows/test-archive.sh b/ci/scripts/windows/test-archive.sh
new file mode 100755
index 0000000..395e33c
--- /dev/null
+++ b/ci/scripts/windows/test-archive.sh
@@ -0,0 +1,130 @@
+#!/usr/local/bin/tini-wrapper /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -x
+
+export TERM=${TERM:-dumb}
+export BUILDROOT=$(pwd)
+export DEST_DIR=${BUILDROOT}/built-geode
+export GRADLE_TASK=${1}
+export SANITIZED_GRADLE_TASK=${GRADLE_TASK##*:}
+export BASE_FILENAME=${2}
+export TMPDIR=${DEST_DIR}/tmp
+export GEODE_BUILD=${DEST_DIR}/test
+export GEODE_BUILD_VERSION_NUMBER=$(grep "versionNumber *=" ${GEODE_BUILD}/gradle.properties | awk -F "=" '{print $2}' | tr -d ' ')
+export BUILD_TIMESTAMP=$(date +%s)
+GEODE_PULL_REQUEST_ID_FILE=${BUILDROOT}/geode/.git/id
+if [ -e "${GEODE_PULL_REQUEST_ID_FILE}" ]; then
+ GEODE_PULL_REQUEST_ID=$(cat ${GEODE_PULL_REQUEST_ID_FILE})
+fi
+
+GCLOUD=gcloud
+GSUTIL=gsutil
+UNAME_O=$(uname -o)
+# Check for Windows (MINGW64) environment
+if [ ${UNAME_O} = "Msys" ]; then
+ GCLOUD=gcloud.cmd
+ GSUTIL=gsutil.cmd
+fi
+
+
+GEODE_BUILD_VERSION_FILE=${BUILDROOT}/geode-build-version/number
+
+if [ ! -e "${GEODE_BUILD_VERSION_FILE}" ] && [ -z "${GEODE_PULL_REQUEST_ID}" ]; then
+ echo "${GEODE_BUILD_VERSION_FILE} file does not exist. Concourse is probably not configured correctly."
+ exit 1
+fi
+if [ -z ${MAINTENANCE_VERSION+x} ]; then
+ echo "MAINTENANCE_VERSION is unset. Check your pipeline configuration and make sure this script is called properly."
+ exit 1
+fi
+
+
+if [ -z "${GEODE_PULL_REQUEST_ID}" ]; then
+CONCOURSE_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE})
+ CONCOURSE_PRODUCT_VERSION=${CONCOURSE_VERSION%%-*}
+ GEODE_PRODUCT_VERSION=${GEODE_BUILD_VERSION_NUMBER}
+ CONCOURSE_BUILD_SLUG=${CONCOURSE_VERSION##*-}
+ BUILD_ID=${CONCOURSE_VERSION##*.}
+ FULL_PRODUCT_VERSION=${GEODE_PRODUCT_VERSION}-${CONCOURSE_BUILD_SLUG}
+
+ echo "Concourse VERSION is ${CONCOURSE_VERSION}"
+ echo "Geode product VERSION is ${GEODE_PRODUCT_VERSION}"
+ echo "Build ID is ${BUILD_ID}"
+else
+ FULL_PRODUCT_VERSION="geode-pr-${GEODE_PULL_REQUEST_ID}"
+fi
+
+
+directories_file=${DEST_DIR}/artifact_directories
+mkdir -p ${TMPDIR}
+
+echo "TMPDIR = ${TMPDIR}"
+echo "GRADLE_TASK = ${GRADLE_TASK}"
+echo "BASE_FILENAME = ${BASE_FILENAME}"
+
+set +e
+$GCLOUD info
+$GCLOUD config set account ${SERVICE_ACCOUNT}
+
+$GSUTIL ls gs://files.apachegeode-ci.info/
+set -e
+
+export FILENAME=${BASE_FILENAME}-${FULL_PRODUCT_VERSION}.tgz
+
+pushd ${GEODE_BUILD}
+
+ set +e
+ ./gradlew combineReports
+ find . -type d -name "reports" > ${directories_file}
+ find . -type d -name "test-results" >> ${directories_file}
+ (find . -type d -name "*Test" | grep "build/[^/]*Test$") >> ${directories_file}
+ find . -name "*-progress*txt" >> ${directories_file}
+ find . -type d -name "callstacks" >> ${directories_file}
+ echo "Collecting the following artifacts..."
+ cat ${directories_file}
+ echo ""
+ tar zcf ${DEST_DIR}/${FILENAME} -T ${directories_file}
+popd
+
+ARTIFACTS_DESTINATION="${PUBLIC_BUCKET}/builds/${FULL_PRODUCT_VERSION}"
+TEST_RESULTS_DESTINATION="${ARTIFACTS_DESTINATION}/test-results/${SANITIZED_GRADLE_TASK}/${BUILD_TIMESTAMP}/"
+TEST_ARTIFACTS_DESTINATION="${ARTIFACTS_DESTINATION}/test-artifacts/${BUILD_TIMESTAMP}/"
+
+
+if [ ! -d "${GEODE_BUILD}/build/reports/combined" ]; then
+ echo "No tests exist, compile failed."
+ mkdir -p ${GEODE_BUILD}/build/reports/combined
+ echo "<html><head><title>No Test Results Were Captured</title></head><body><h1>No Test Results Were Captured</h1></body></html>" > ${GEODE_BUILD}/build/reports/combined/index.html
+fi
+
+pushd ${GEODE_BUILD}/build/reports/combined
+$GSUTIL -q -m cp -r * gs://${TEST_RESULTS_DESTINATION}
+popd
+
+echo ""
+printf "\033[92m=-=-=-=-=-=-=-=-=-=-=-=-=-= Test Results Website =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\033[0m\n"
+printf "\033[92mhttp://${TEST_RESULTS_DESTINATION}\033[0m\n"
+printf "\033[92m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\033[0m\n"
+printf "\n"
+
+$GSUTIL cp ${DEST_DIR}/${FILENAME} gs://${TEST_ARTIFACTS_DESTINATION}
+
+printf "\033[92mTest artifacts from this job are available at:\033[0m\n"
+printf "\n"
+printf "\033[92mhttp://${TEST_ARTIFACTS_DESTINATION}${FILENAME}\033[0m\n"
diff --git a/ci/scripts/windows/test-run.sh b/ci/scripts/windows/test-run.sh
new file mode 100755
index 0000000..43c6cf3
--- /dev/null
+++ b/ci/scripts/windows/test-run.sh
@@ -0,0 +1,94 @@
+#!/usr/local/bin/tini-wrapper /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+export TERM=${TERM:-dumb}
+export BUILDROOT=$(pwd)
+export DEST_DIR=${BUILDROOT}/built-geode
+export GRADLE_TASK=${1}
+export BASE_FILENAME=${2}
+export GRADLE_TEST_CATEGORY=${3}
+export GEODE_BUILD=${DEST_DIR}/test
+
+printf "\nUsing the following JDK:"
+if [ -n "${JAVA_HOME}" ]; then
+ ${JAVA_HOME}/bin/java -version
+else
+ java -version
+fi
+printf "\n\n"
+
+directories_file=${DEST_DIR}/artifact_directories
+
+echo "GRADLE_TASK = ${GRADLE_TASK}"
+echo "BASE_FILENAME = ${BASE_FILENAME}"
+
+DEFAULT_GRADLE_TASK_OPTIONS="--no-daemon -x javadoc -x spotlessCheck"
+
+if [[ -n "${GRADLE_TEST_CATEGORY}" ]]; then
+ GRADLE_TASK_OPTIONS="-PtestCategory=${GRADLE_TEST_CATEGORY}"
+fi
+
+mkdir -p ${GEODE_BUILD}
+if [ -v CALL_STACK_TIMEOUT ]; then
+ geode-ci/ci/scripts/capture-call-stacks.sh ${CALL_STACK_TIMEOUT} &
+fi
+
+pushd geode
+ tar cf - * | (cd ${GEODE_BUILD}; tar xpf -)
+popd
+
+if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then
+ PARALLEL_DUNIT="-PparallelDunit"
+ if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then
+ DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}"
+ fi
+else
+ PARALLEL_DUNIT=""
+ DUNIT_PARALLEL_FORKS=""
+fi
+
+pushd ${GEODE_BUILD}
+ set +e
+ echo "Running tests"
+ set -x
+
+# ./gradlew --no-daemon -x javadoc -x spotlessCheck :geode-assembly:acceptanceTest --tests org.apache.geode.management.internal.cli.commands.PutCommandWithJsonTest
+ ./gradlew ${PARALLEL_DUNIT} \
+ ${DUNIT_PARALLEL_FORKS} \
+ ${DUNIT_DOCKER_IMAGE} \
+ ${DEFAULT_GRADLE_TASK_OPTIONS} \
+ ${GRADLE_TASK_OPTIONS} \
+ ${GRADLE_TASK}
+ export GRADLE_EXIT_STATUS=$?
+ set +x
+popd
+
+echo "*************************************************************"
+echo "Results information is located in the 'archive-results' task"
+echo "*************************************************************"
+
+echo "GRADLE_EXIT_STATUS is ${GRADLE_EXIT_STATUS}"
+
+
+if [[ "${GRADLE_EXIT_STATUS}" != "0" && "${GRADLE_TASK}" == "test" ]]; then
+ error_exit
+fi
+exit ${GRADLE_EXIT_STATUS}