You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by je...@apache.org on 2018/08/24 15:09:50 UTC
[geode] 01/06: initial commit for windows heavy lifting
This is an automated email from the ASF dual-hosted git repository.
jensdeppe pushed a commit to branch windows-heavy-lifter
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 97a7795da3edf4cfb2460582b63c85332fb0a929
Author: Jens Deppe <jd...@pivotal.io>
AuthorDate: Wed Aug 22 11:58:01 2018 -0700
initial commit for windows heavy lifting
---
ci/pipelines/geode-build/jinja.template.yml | 71 +++--------------------------
ci/pipelines/shared/jinja.variables.yml | 53 ++++++++++-----------
ci/scripts/execute_tests.sh | 2 +-
ci/scripts/start_instance.sh | 29 ++++++++++--
4 files changed, 56 insertions(+), 99 deletions(-)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 39acf0a..7762377 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -31,7 +31,12 @@
{% endmacro %}
{% macro common_test_params(test) %}
+ {%- if test.CALL_STACK_TIMEOUT -%}
CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+ {%- endif -%}
+ {%- if test.GRADLE_TASK_OPTIONS -%}
+GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}}
+ {%- endif -%}
DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
MAINTENANCE_VERSION: {{repository.branch}}
PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
@@ -43,7 +48,7 @@ groups:
- name: main
jobs:
- Build
- {% for test in (tests + windowstests) if not test.name=="StressNew" -%}
+ {% for test in tests if not test.name=="StressNew" -%}
- {{test.name}}Test
{% endfor -%}
- UpdatePassingRef
@@ -283,67 +288,3 @@ jobs:
- name: instance-data
timeout: 1h
{% endfor -%}
-{% for test in windowstests %}
-- name: {{test.name}}Test
- public: true
- serial: true
- plan:
- {{ plan_resource_gets() |indent(2) }}
- - task: cleanup-java-processes
- config:
- platform: windows
- run:
- path: powershell
- args:
- - -command
- - |
- gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
- kill -name java -force
- exit 0
- tags:
- - windows-{{test.tags}}-tests
- - task: execute_tests
- config:
- platform: windows
- params:
- JAVA_HOME: C:\progra~1\java\jdk1.8.0_181
- SOURCE_REPOSITORY: {{repository.branch}}
- {{ common_test_params(test) | indent(8) }}
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-run.sh
- - {{test.target_arg}}
- - {{test.dir_arg}}
- - {{test.package_arg}}
- inputs:
- - name: geode-ci
- - name: geode
- outputs:
- - name: built-geode
- tags:
- - windows-{{test.tags}}-tests
- ensure:
- aggregate:
- - task: archive_results
- config:
- platform: windows
- params:
- MAINTENANCE_VERSION: {{repository.branch}}
- PUBLIC_BUCKET: ((!public-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-archive.sh
- - {{test.target_arg}}
- - {{test.dir_arg}}
- inputs:
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- tags:
- - windows-{{test.tags}}-tests
- timeout: 1h
- timeout: 6h
-{% endfor %}
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index fd4789e..d0a28f4 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -18,12 +18,6 @@
repository:
project: geode
-platforms:
-- name: "Linux"
- prefix: ""
-- name: "Windows"
- prefix: "Windows"
-
tests:
- name: "Acceptance"
CPUS: "8"
@@ -70,37 +64,40 @@ tests:
GRADLE_TASK: repeatTest
execute_test_timeout: 2h15m
PARALLEL_DUNIT: "true"
-
-windowstests:
- name: "WindowsAcceptance"
- tags: "acceptance"
- CALL_STACK_TIMEOUT: "1800"
+ tags: "windows"
+ CPUS: "16"
+ RAM: "32"
+ ARTIFACT_SLUG: windows-acceptancetestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: :geode-assembly:acceptanceTest
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: :geode-assembly:acceptanceTest
- dir_arg: windows-acceptancetestfiles
- package_arg: ""
- name: "WindowsGfshDistributed"
- tags: "distributed"
- CALL_STACK_TIMEOUT: "1800"
+ tags: "windows"
+ CPUS: "16"
+ RAM: "32"
+ ARTIFACT_SLUG: windows-gfshdistributedtest
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: distributedTest
+ GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest"
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: distributedTest
- dir_arg: windows-gfshdistributedtest
- package_arg: org.apache.geode.test.junit.categories.GfshTest
- name: "WindowsIntegration"
- tags: "integration"
- CALL_STACK_TIMEOUT: "1800"
+ tags: "windows"
+ CPUS: "16"
+ RAM: "32"
+ ARTIFACT_SLUG: windows-integrationtestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: integrationTest
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: integrationTest
- dir_arg: windows-integrationtestfiles
- package_arg: ""
- name: "WindowsUnit"
- tags: "unit"
- CALL_STACK_TIMEOUT: "1800"
+ tags: "windows"
+ CPUS: "16"
+ RAM: "32"
+ ARTIFACT_SLUG: windows-unittestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: test
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: test
- dir_arg: windows-unittestfiles
- package_arg: ""
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index e270994..00db410 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -79,4 +79,4 @@ GRADLE_COMMAND="./gradlew \
${SCM_PROPS}"
echo "${GRADLE_COMMAND}"
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${GRADLE_COMMAND}"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'"
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index c9a54f2..aa85924 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -50,11 +50,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-" | tr '[
SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
IMAGE_FAMILY_PREFIX=""
+WINDOWS_PREFIX=""
if [[ "${GEODE_FORK}" != "apache" ]]; then
IMAGE_FAMILY_PREFIX="${GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
fi
+if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ Windows ]]; then
+ WINDOWS_PREFIX="windows-"
+fi
+
INSTANCE_NAME="$(echo "build-${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]' '[:lower:]')"
PROJECT=apachegeode-ci
ZONE=us-central1-f
@@ -76,7 +81,7 @@ while true; do
--min-cpu-platform=Intel\ Skylake \
--network="heavy-lifters" \
--subnet="heavy-lifters" \
- --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \
+ --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \
--image-project=${PROJECT} \
--boot-disk-size=100GB \
--boot-disk-type=pd-ssd \
@@ -96,9 +101,23 @@ done
echo "${INSTANCE_INFORMATION}" > instance-data/instance-information
-INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP')
+# This extracts the internal IP address for subsequent use
+INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].networkIP')
echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address"
-while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
- echo -n .
-done
+if [[ -z "${WINDOWS_PREFIX}" ]]; then
+ while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
+ echo -n .
+ done
+else
+ # Set up ssh access for Windows systems
+ while [[ -z "${PASSWORD}" ]]; do
+ PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode --zone=${ZONE} --format json | jq -r .password )
+ sleep 5
+ done
+
+ KEY=$( cat ${SSHKEY_FILE} )
+
+ winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \
+ "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys -encoding utf8 -value '${KEY}' }\""
+fi