[beam] branch master updated: [BEAM-4122] Tune Gradle resource usage on Jenkins (#5171)

[al...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new c929703  [BEAM-4122] Tune Gradle resource usage on Jenkins (#5171)
c929703 is described below

commit c929703eec1c8a9825d4ac9f439ee3f1bcdeb54b
Author: Scott Wegner <sw...@gmail.com>
AuthorDate: Thu Apr 19 09:27:40 2018 -0700

    [BEAM-4122] Tune Gradle resource usage on Jenkins (#5171)
    
    * Limit the number of Gradle workers on Jenkins
    * Remove Dataflow Validates Runner --max-workers setting.
    Our Gradle builds are frequently exhausting memory on the Jenkins lab
    machines. We have capped the number of workers at 2 across all jobs to
    limit the total memory usage per job.
    * Enable build caching
---
 .test-infra/jenkins/common_job_properties.groovy                 | 9 ++++-----
 .../job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy     | 6 ------
 gradle.properties                                                | 1 +
 3 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
index d19fc88..1ab3a2f 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -170,11 +170,10 @@ class common_job_properties {
     "--info",
     // Continue the build even if there is a failure to show as many potential failures as possible.
     '--continue',
-    // Until we verify the build cache is working appropriately, force rerunning all tasks
-    '--rerun-tasks',
-    // Disable daemon, which helps ensure hermetic environment at small startup performance penalty.
-    // This needs to be disabled if we move to incremental builds.
-    "--no-daemon",
+    // Limit background number of workers to prevent exhausting machine memory.
+    // Jenkins machines have 15GB memory, and run 2 jobs in parallel; workers are configured with
+    // JVM max heap size 3.5GB. So 2 jobs * 2 workers * 3.5GB heap = 14GB
+    '--max-workers=2',
   ]
 
   static void setGradleSwitches(context) {
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
index 31d3685..07f3b6c 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
@@ -48,12 +48,6 @@ job('beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle') {
       rootBuildScriptDir(common_job_properties.checkoutDir)
       tasks(':beam-runners-google-cloud-dataflow-java:validatesRunner')
       common_job_properties.setGradleSwitches(delegate)
-      // Increase parallel worker threads above processor limit since most time is
-      // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-      // because each one launches a Dataflow job with about 3 mins of overhead.
-      // 3 x num_cores strikes a good balance between maxing out parallelism without
-      // overloading the machines.
-      switches("--max-workers=${3 * Runtime.runtime.availableProcessors()}")
     }
   }
 }
diff --git a/gradle.properties b/gradle.properties
index 4b34251..c0034d2 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -15,6 +15,7 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
+org.gradle.caching=true
 org.gradle.parallel=true
 org.gradle.configureondemand=true
 offlineRepositoryRoot=offline-repository

-- 
To stop receiving notification emails like this one, please contact
altay@apache.org.