You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by je...@apache.org on 2018/08/30 19:45:10 UTC
[geode] 01/01: GEODE-5668: Switch Windows build jobs to
'heavy-lifter' model
This is an automated email from the ASF dual-hosted git repository.
jensdeppe pushed a commit to branch windows-heavy-lifter
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 62d84b790c7021cf8f98166678538005d4f61d5a
Author: Jens Deppe <jd...@pivotal.io>
AuthorDate: Wed Aug 22 11:58:01 2018 -0700
GEODE-5668: Switch Windows build jobs to 'heavy-lifter' model
---
ci/pipelines/geode-build/jinja.template.yml | 71 +++--------------------------
ci/pipelines/shared/jinja.variables.yml | 49 +++++++++-----------
ci/scripts/execute_tests.sh | 2 +-
ci/scripts/rsync_code_down.sh | 4 +-
ci/scripts/start_instance.sh | 34 +++++++++++---
gradle.properties | 2 +-
6 files changed, 59 insertions(+), 103 deletions(-)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index e94c99d..e72b2ca 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -31,7 +31,12 @@
{% endmacro %}
{% macro common_test_params(test) %}
+ {% if test.CALL_STACK_TIMEOUT %}
CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+ {% endif %}
+ {% if test.GRADLE_TASK_OPTIONS %}
+GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}}
+ {% endif %}
DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
MAINTENANCE_VERSION: {{repository.branch}}
PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
@@ -44,7 +49,7 @@ groups:
- name: main
jobs:
- Build
- {% for test in (tests + windowstests) if not test.name=="StressNew" -%}
+ {% for test in tests if not test.name=="StressNew" -%}
- {{test.name}}Test
{% endfor -%}
- UpdatePassingRef
@@ -321,67 +326,3 @@ jobs:
- name: instance-data
timeout: 1h
{% endfor -%}
-{% for test in windowstests %}
-- name: {{test.name}}Test
- public: true
- serial: true
- plan:
- {{ plan_resource_gets() |indent(2) }}
- - task: cleanup-java-processes
- config:
- platform: windows
- run:
- path: powershell
- args:
- - -command
- - |
- gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
- kill -name java -force
- exit 0
- tags:
- - windows-{{test.tags}}-tests
- - task: execute_tests
- config:
- platform: windows
- params:
- JAVA_HOME: C:\progra~1\java\jdk1.8.0_181
- SOURCE_REPOSITORY: {{repository.branch}}
- {{ common_test_params(test) | indent(8) }}
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-run.sh
- - {{test.target_arg}}
- - {{test.dir_arg}}
- - {{test.package_arg}}
- inputs:
- - name: geode-ci
- - name: geode
- outputs:
- - name: built-geode
- tags:
- - windows-{{test.tags}}-tests
- ensure:
- aggregate:
- - task: archive_results
- config:
- platform: windows
- params:
- MAINTENANCE_VERSION: {{repository.branch}}
- PUBLIC_BUCKET: ((!public-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-archive.sh
- - {{test.target_arg}}
- - {{test.dir_arg}}
- inputs:
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- tags:
- - windows-{{test.tags}}-tests
- timeout: 1h
- timeout: 6h
-{% endfor %}
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index 85d495a..e37c7ee 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -18,12 +18,6 @@
repository:
project: geode
-platforms:
-- name: "Linux"
- prefix: ""
-- name: "Windows"
- prefix: "Windows"
-
tests:
- name: "Acceptance"
CPUS: "8"
@@ -71,37 +65,36 @@ tests:
GRADLE_TASK: repeatTest
execute_test_timeout: 2h15m
PARALLEL_DUNIT: "true"
-
-windowstests:
- name: "WindowsAcceptance"
- tags: "acceptance"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-acceptancetestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: :geode-assembly:acceptanceTest
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: :geode-assembly:acceptanceTest
- dir_arg: windows-acceptancetestfiles
- package_arg: ""
- name: "WindowsGfshDistributed"
- tags: "distributed"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-gfshdistributedtest
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: distributedTest
+ GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest"
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: distributedTest
- dir_arg: windows-gfshdistributedtest
- package_arg: org.apache.geode.test.junit.categories.GfshTest
- name: "WindowsIntegration"
- tags: "integration"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-integrationtestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: integrationTest
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: integrationTest
- dir_arg: windows-integrationtestfiles
- package_arg: ""
- name: "WindowsUnit"
- tags: "unit"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-unittestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: test
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: test
- dir_arg: windows-unittestfiles
- package_arg: ""
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index d36f85b..c4bd456 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -84,4 +84,4 @@ GRADLE_COMMAND="./gradlew \
${SCM_PROPS}"
echo "${GRADLE_COMMAND}"
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${GRADLE_COMMAND}"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'"
diff --git a/ci/scripts/rsync_code_down.sh b/ci/scripts/rsync_code_down.sh
index ce12bcc..a94bb90 100755
--- a/ci/scripts/rsync_code_down.sh
+++ b/ci/scripts/rsync_code_down.sh
@@ -40,7 +40,7 @@ echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
OUTPUT_DIR=${BASE_DIR}/geode-results
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "cd geode && ./gradlew combineReports"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'cd geode; ./gradlew --no-daemon combineReports'"
-time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:. ${OUTPUT_DIR}/.
+time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:geode ${OUTPUT_DIR}/
set +x
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index 3bee7f9..cd4d191 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -18,6 +18,7 @@
# limitations under the License.
set -e
+set -x
BASE_DIR=$(pwd)
@@ -54,11 +55,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-" | tr '[
SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
IMAGE_FAMILY_PREFIX=""
+WINDOWS_PREFIX=""
if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
fi
+if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then
+ WINDOWS_PREFIX="windows-"
+fi
+
INSTANCE_NAME="$(echo "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]' '[:lower:]')"
PROJECT=apachegeode-ci
ZONE=us-central1-f
@@ -78,9 +84,9 @@ while true; do
--zone=${ZONE} \
--machine-type=custom-${CPUS}-${RAM_MEGABYTES} \
--min-cpu-platform=Intel\ Skylake \
- --network="heavy-lifters" \
- --subnet="heavy-lifters" \
- --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \
+ --network="heavy-lifters-2" \
+ --subnet="heavy-lifters-2" \
+ --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \
--image-project=${PROJECT} \
--boot-disk-size=100GB \
--boot-disk-type=pd-ssd \
@@ -103,6 +109,22 @@ echo "${INSTANCE_INFORMATION}" > instance-data/instance-information
INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP')
echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address"
-while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
- echo -n .
-done
+if [[ -z "${WINDOWS_PREFIX}" ]]; then
+ while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
+ echo -n .
+ done
+else
+ # Set up ssh access for Windows systems
+ while [[ -z "${PASSWORD}" ]]; do
+ PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode --zone=${ZONE} --format json | jq -r .password )
+ sleep 5
+ done
+
+ ssh-keygen -N "" -f ${SSHKEY_FILE}
+
+ KEY=$( cat ${SSHKEY_FILE}.pub )
+
+ winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \
+ -https -insecure -port 5986 \
+ "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys -encoding utf8 -value '${KEY}' }\""
+fi
diff --git a/gradle.properties b/gradle.properties
index 3edf13d..e9c97ec 100755
--- a/gradle.properties
+++ b/gradle.properties
@@ -42,7 +42,7 @@ productName = Apache Geode
productOrg = Apache Software Foundation (ASF)
org.gradle.daemon = true
-org.gradle.jvmargs = -Xmx3g
+org.gradle.jvmargs = -Xmx4g
minimumGradleVersion = 4.10
# Set this on the command line with -P or in ~/.gradle/gradle.properties