You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by je...@apache.org on 2018/08/30 19:45:09 UTC
[geode] branch windows-heavy-lifter updated (8e8991a -> 62d84b7)
This is an automated email from the ASF dual-hosted git repository.
jensdeppe pushed a change to branch windows-heavy-lifter
in repository https://gitbox.apache.org/repos/asf/geode.git.
discard 8e8991a Don't need ssh timing fixes
discard ef8c342 Back to public IPs
discard fb58a20 Use private IP addresses
discard 6e86711 Use heavy-lifters-2 network
discard 2bca4fc initial commit for windows heavy lifting
add 64845ab GEODE-5594: Rename SSL hostname validation property name (#2398)
add 574530c GEODE-5536: Force unitTest compilation to be dependency of source classes (#2337)
add 9e4912f GEODE-5662: improve timeouts in ExecutorServiceRuleTest
add ff56282 GEODE-5645 Client cache misses invalidate
add c6fe1df GEODE-5596 Client ends up with destroyed entry after invalidate()
add 14cbd3e Revert "GEODE-5596 Client ends up with destroyed entry after invalidate()"
add 623d4de GEODE-5596 Client ends up with destroyed entry after invalidate()
add 9ee8cbf GEODE-3780 member is not considered suspect after failing final check
add 06ba32c GEODE-5475: rule improvement
add e76f1e3 GEODE-5594 Enable endpoint validation during using SSL handshake (#2406)
add 52a6e42 GEODE-5652: use unlimited ThreadPoolExecutor in ExecutorServiceRule
new 62d84b7 GEODE-5668: Switch Windows build jobs to 'heavy-lifter' model
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (8e8991a)
\
N -- N -- N refs/heads/windows-heavy-lifter (62d84b7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ci/pipelines/shared/jinja.variables.yml | 12 +-
.../geode/cache30/ClientServerCCEDUnitTest.java | 72 +++++-
.../cache/PartitionedRegionCreationDUnitTest.java | 1 -
.../tier/sockets/ClientServerMiscDUnitTest.java | 86 +++++++
.../internal/locks/DLockServiceLeakTest.java | 5 +-
.../gms/fd/GMSHealthMonitorJUnitTest.java | 16 ++
.../apache/geode/internal/SSLConfigJUnitTest.java | 19 ++
.../internal/cache/InterruptDiskJUnitTest.java | 120 ++++------
.../org/apache/geode/cache/AttributesFactory.java | 2 +-
.../geode/distributed/ConfigurationProperties.java | 2 +-
.../membership/gms/fd/GMSHealthMonitor.java | 79 +++----
.../geode/internal/cache/AbstractRegionMap.java | 37 ++-
.../apache/geode/internal/cache/LocalRegion.java | 4 +-
.../geode/cache/AttributesFactoryJUnitTest.java | 1 +
.../managing/security/implementing_ssl.html.md.erb | 12 +-
.../topics/gemfire_properties.html.md.erb | 9 +
.../sockets/ClientServerMiscDUnitTestBase.java | 44 ++--
.../geode/test/junit/rules/ServerStarterRule.java | 29 ++-
.../rules/ExecutorServiceRuleIntegrationTest.java | 10 +-
.../test/junit/rules/ExecutorServiceRule.java | 68 +-----
.../test/junit/rules/ExecutorServiceRuleTest.java | 260 +++------------------
gradle/test.gradle | 1 +
22 files changed, 404 insertions(+), 485 deletions(-)
[geode] 01/01: GEODE-5668: Switch Windows build jobs to
'heavy-lifter' model
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jensdeppe pushed a commit to branch windows-heavy-lifter
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 62d84b790c7021cf8f98166678538005d4f61d5a
Author: Jens Deppe <jd...@pivotal.io>
AuthorDate: Wed Aug 22 11:58:01 2018 -0700
GEODE-5668: Switch Windows build jobs to 'heavy-lifter' model
---
ci/pipelines/geode-build/jinja.template.yml | 71 +++--------------------------
ci/pipelines/shared/jinja.variables.yml | 49 +++++++++-----------
ci/scripts/execute_tests.sh | 2 +-
ci/scripts/rsync_code_down.sh | 4 +-
ci/scripts/start_instance.sh | 34 +++++++++++---
gradle.properties | 2 +-
6 files changed, 59 insertions(+), 103 deletions(-)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index e94c99d..e72b2ca 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -31,7 +31,12 @@
{% endmacro %}
{% macro common_test_params(test) %}
+ {% if test.CALL_STACK_TIMEOUT %}
CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+ {% endif %}
+ {% if test.GRADLE_TASK_OPTIONS %}
+GRADLE_TASK_OPTIONS: {{test.GRADLE_TASK_OPTIONS}}
+ {% endif %}
DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
MAINTENANCE_VERSION: {{repository.branch}}
PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
@@ -44,7 +49,7 @@ groups:
- name: main
jobs:
- Build
- {% for test in (tests + windowstests) if not test.name=="StressNew" -%}
+ {% for test in tests if not test.name=="StressNew" -%}
- {{test.name}}Test
{% endfor -%}
- UpdatePassingRef
@@ -321,67 +326,3 @@ jobs:
- name: instance-data
timeout: 1h
{% endfor -%}
-{% for test in windowstests %}
-- name: {{test.name}}Test
- public: true
- serial: true
- plan:
- {{ plan_resource_gets() |indent(2) }}
- - task: cleanup-java-processes
- config:
- platform: windows
- run:
- path: powershell
- args:
- - -command
- - |
- gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
- kill -name java -force
- exit 0
- tags:
- - windows-{{test.tags}}-tests
- - task: execute_tests
- config:
- platform: windows
- params:
- JAVA_HOME: C:\progra~1\java\jdk1.8.0_181
- SOURCE_REPOSITORY: {{repository.branch}}
- {{ common_test_params(test) | indent(8) }}
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-run.sh
- - {{test.target_arg}}
- - {{test.dir_arg}}
- - {{test.package_arg}}
- inputs:
- - name: geode-ci
- - name: geode
- outputs:
- - name: built-geode
- tags:
- - windows-{{test.tags}}-tests
- ensure:
- aggregate:
- - task: archive_results
- config:
- platform: windows
- params:
- MAINTENANCE_VERSION: {{repository.branch}}
- PUBLIC_BUCKET: ((!public-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-archive.sh
- - {{test.target_arg}}
- - {{test.dir_arg}}
- inputs:
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- tags:
- - windows-{{test.tags}}-tests
- timeout: 1h
- timeout: 6h
-{% endfor %}
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index 85d495a..e37c7ee 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -18,12 +18,6 @@
repository:
project: geode
-platforms:
-- name: "Linux"
- prefix: ""
-- name: "Windows"
- prefix: "Windows"
-
tests:
- name: "Acceptance"
CPUS: "8"
@@ -71,37 +65,36 @@ tests:
GRADLE_TASK: repeatTest
execute_test_timeout: 2h15m
PARALLEL_DUNIT: "true"
-
-windowstests:
- name: "WindowsAcceptance"
- tags: "acceptance"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-acceptancetestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: :geode-assembly:acceptanceTest
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: :geode-assembly:acceptanceTest
- dir_arg: windows-acceptancetestfiles
- package_arg: ""
- name: "WindowsGfshDistributed"
- tags: "distributed"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-gfshdistributedtest
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: distributedTest
+ GRADLE_TASK_OPTIONS: "-PtestCategory=org.apache.geode.test.junit.categories.GfshTest"
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: distributedTest
- dir_arg: windows-gfshdistributedtest
- package_arg: org.apache.geode.test.junit.categories.GfshTest
- name: "WindowsIntegration"
- tags: "integration"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-integrationtestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: integrationTest
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: integrationTest
- dir_arg: windows-integrationtestfiles
- package_arg: ""
- name: "WindowsUnit"
- tags: "unit"
- CALL_STACK_TIMEOUT: "1800"
+ CPUS: "16"
+ RAM: "64"
+ ARTIFACT_SLUG: windows-unittestfiles
DUNIT_PARALLEL_FORKS: "0"
+ GRADLE_TASK: test
+ execute_test_timeout: 6h
PARALLEL_DUNIT: "false"
- target_arg: test
- dir_arg: windows-unittestfiles
- package_arg: ""
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index d36f85b..c4bd456 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -84,4 +84,4 @@ GRADLE_COMMAND="./gradlew \
${SCM_PROPS}"
echo "${GRADLE_COMMAND}"
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${GRADLE_COMMAND}"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'mkdir -p tmp; cd geode; ${GRADLE_COMMAND}'"
diff --git a/ci/scripts/rsync_code_down.sh b/ci/scripts/rsync_code_down.sh
index ce12bcc..a94bb90 100755
--- a/ci/scripts/rsync_code_down.sh
+++ b/ci/scripts/rsync_code_down.sh
@@ -40,7 +40,7 @@ echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
OUTPUT_DIR=${BASE_DIR}/geode-results
-ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "cd geode && ./gradlew combineReports"
+ssh -i ${SSHKEY_FILE} geode@${INSTANCE_IP_ADDRESS} "bash -c 'cd geode; ./gradlew --no-daemon combineReports'"
-time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:. ${OUTPUT_DIR}/.
+time rsync -e "ssh -i ${SSHKEY_FILE}" -ah geode@${INSTANCE_IP_ADDRESS}:geode ${OUTPUT_DIR}/
set +x
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/start_instance.sh
index 3bee7f9..cd4d191 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/start_instance.sh
@@ -18,6 +18,7 @@
# limitations under the License.
set -e
+set -x
BASE_DIR=$(pwd)
@@ -54,11 +55,16 @@ SANITIZED_BUILD_PIPELINE_NAME=$(echo ${BUILD_PIPELINE_NAME} | tr "/" "-" | tr '[
SANITIZED_BUILD_JOB_NAME=$(echo ${BUILD_JOB_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
SANITIZED_BUILD_NAME=$(echo ${BUILD_NAME} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
IMAGE_FAMILY_PREFIX=""
+WINDOWS_PREFIX=""
if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
fi
+if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then
+ WINDOWS_PREFIX="windows-"
+fi
+
INSTANCE_NAME="$(echo "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" | tr '[:upper:]' '[:lower:]')"
PROJECT=apachegeode-ci
ZONE=us-central1-f
@@ -78,9 +84,9 @@ while true; do
--zone=${ZONE} \
--machine-type=custom-${CPUS}-${RAM_MEGABYTES} \
--min-cpu-platform=Intel\ Skylake \
- --network="heavy-lifters" \
- --subnet="heavy-lifters" \
- --image-family="${IMAGE_FAMILY_PREFIX}geode-builder" \
+ --network="heavy-lifters-2" \
+ --subnet="heavy-lifters-2" \
+ --image-family="${IMAGE_FAMILY_PREFIX}${WINDOWS_PREFIX}geode-builder" \
--image-project=${PROJECT} \
--boot-disk-size=100GB \
--boot-disk-type=pd-ssd \
@@ -103,6 +109,22 @@ echo "${INSTANCE_INFORMATION}" > instance-data/instance-information
INSTANCE_IP_ADDRESS=$(echo ${INSTANCE_INFORMATION} | jq -r '.[].networkInterfaces[0].accessConfigs[0].natIP')
echo "${INSTANCE_IP_ADDRESS}" > "instance-data/instance-ip-address"
-while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
- echo -n .
-done
+if [[ -z "${WINDOWS_PREFIX}" ]]; then
+ while ! gcloud compute --project=${PROJECT} ssh geode@${INSTANCE_NAME} --zone=${ZONE} --ssh-key-file=${SSHKEY_FILE} --quiet -- true; do
+ echo -n .
+ done
+else
+ # Set up ssh access for Windows systems
+ while [[ -z "${PASSWORD}" ]]; do
+ PASSWORD=$( yes | gcloud beta compute reset-windows-password ${INSTANCE_NAME} --user=geode --zone=${ZONE} --format json | jq -r .password )
+ sleep 5
+ done
+
+ ssh-keygen -N "" -f ${SSHKEY_FILE}
+
+ KEY=$( cat ${SSHKEY_FILE}.pub )
+
+ winrm -hostname ${INSTANCE_IP_ADDRESS} -username geode -password "${PASSWORD}" \
+ -https -insecure -port 5986 \
+ "powershell -command \"&{ mkdir c:\users\geode\.ssh -force; set-content -path c:\users\geode\.ssh\authorized_keys -encoding utf8 -value '${KEY}' }\""
+fi
diff --git a/gradle.properties b/gradle.properties
index 3edf13d..e9c97ec 100755
--- a/gradle.properties
+++ b/gradle.properties
@@ -42,7 +42,7 @@ productName = Apache Geode
productOrg = Apache Software Foundation (ASF)
org.gradle.daemon = true
-org.gradle.jvmargs = -Xmx3g
+org.gradle.jvmargs = -Xmx4g
minimumGradleVersion = 4.10
# Set this on the command line with -P or in ~/.gradle/gradle.properties