You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2019/12/05 10:14:19 UTC
[lucene-solr] branch gradle-master updated: Generate
hardware-specific defaults for gradle parallelism on the first build run
(any task). Add some explanations on how to tweak local settings even
further (gradlew :helpLocalSettings
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch gradle-master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/gradle-master by this push:
new bf7d115 Generate hardware-specific defaults for gradle parallelism on the first build run (any task). Add some explanations on how to tweak local settings even further (gradlew :helpLocalSettings
bf7d115 is described below
commit bf7d115414c45a532e81cfea4857f9402b0b1e7c
Author: Dawid Weiss <dw...@apache.org>
AuthorDate: Thu Dec 5 11:14:09 2019 +0100
Generate hardware-specific defaults for gradle parallelism on the first build run (any task). Add some explanations on how to tweak local settings even further (gradlew :helpLocalSettings
---
.gitignore | 3 ++
build.gradle | 2 +
gradle.properties | 5 --
gradle/defaults.gradle | 15 ++++++
gradle/generate-defaults.gradle | 73 ++++++++++++++++++++++++++++++
gradle/help.gradle | 4 +-
gradle/testing/defaults-tests.gradle | 3 +-
gradle/testing/randomization.gradle | 17 -------
gradle/testing/slowest-tests-at-end.gradle | 2 +-
help/localSettings.txt | 46 +++++++++++++++++++
10 files changed, 144 insertions(+), 26 deletions(-)
diff --git a/.gitignore b/.gitignore
index 023c827..264d7ca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,6 @@ __pycache__
build/
.gradle/
.idea/
+
+# Ignore the generated local settings file.
+gradle.properties
diff --git a/build.gradle b/build.gradle
index 2d1c561..b5d39d3 100644
--- a/build.gradle
+++ b/build.gradle
@@ -16,6 +16,8 @@ allprojects {
// if the build file is incorrectly written and evaluates something
// eagerly).
+apply from: file('gradle/generate-defaults.gradle')
+
// CI systems.
apply from: file('gradle/ci/buildscan.gradle')
apply from: file('gradle/ci/travis.gradle')
diff --git a/gradle.properties b/gradle.properties
deleted file mode 100644
index a7a04d6..0000000
--- a/gradle.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-systemProp.file.encoding=UTF-8
-
-org.gradle.jvmargs=-Xmx1g
-org.gradle.parallel=true
-org.gradle.priority=low
diff --git a/gradle/defaults.gradle b/gradle/defaults.gradle
index 474826e..1b534c4 100644
--- a/gradle/defaults.gradle
+++ b/gradle/defaults.gradle
@@ -15,4 +15,19 @@ allprojects {
// Artifacts will have names after full gradle project path
// so :solr:core will have solr-core.jar, etc.
project.archivesBaseName = project.path.replaceAll("^:", "").replace(':', '-')
+
+ ext {
+ // Utility method to support passing overrides via -P or -D.
+ propertyOrDefault = { propName, defValue ->
+ def result
+ if (project.hasProperty(propName)) {
+ result = project.getProperty(propName)
+ } else if (System.properties.containsKey(propName)) {
+ result = System.properties.get(propName)
+ } else {
+ result = defValue
+ }
+ return result
+ }
+ }
}
diff --git a/gradle/generate-defaults.gradle b/gradle/generate-defaults.gradle
new file mode 100644
index 0000000..72f0daf
--- /dev/null
+++ b/gradle/generate-defaults.gradle
@@ -0,0 +1,73 @@
+
+// This script tries to guess sensible defaults for gradle parallelism
+// and local machine's resources and save them under 'gradle.properties'.
+
+def hasDefaults = rootProject.file("gradle.properties").exists()
+
+// If we don't have the defaults yet, create them and re-run the build
+// recursively with the same parameters as originally passed.
+//
+// Sadly, the recursive build doesn't seem to pick up the parallelism
+// tweaks from gradle.properties file.
+
+if (!hasDefaults) {
+ configure(rootProject) {
+ task setupLocalDefaultsOnce(type: GradleBuild) {
+ // Approximate a common-sense default for running gradle with parallel
+ // workers: half the count of available cpus but not more than 12.
+ def cpus = Runtime.runtime.availableProcessors()
+ def maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12))
+ def testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 4))
+
+ // Reuse the same set of parameters for the recursive invocation and apply
+ // some of these eagerly.
+ def startParams = gradle.startParameter.newInstance()
+ startParams.setParallelProjectExecutionEnabled(true)
+ startParams.setMaxWorkerCount(maxWorkers)
+ startParameter(startParams)
+
+ // Write the defaults for this machine.
+ rootProject.file("gradle.properties").write(
+ [
+ "# These settings have been generated automatically on the first run.",
+ "# See gradlew :helpLocalSettings for more information.",
+ "systemProp.file.encoding=UTF-8",
+ "org.gradle.daemon=true",
+ "org.gradle.jvmargs=-Xmx1g",
+ "org.gradle.parallel=true",
+ "org.gradle.priority=normal",
+ "",
+ "# Maximum number of parallel gradle workers.",
+ "org.gradle.workers.max=${maxWorkers}",
+ "",
+ "# Maximum number of test JVMs forked per test task.",
+ "tests.jvms=${testsJvms}"
+ ].join("\n"), "UTF-8")
+
+ doFirst {
+ logger.log(LogLevel.WARN, "\nIMPORTANT. This is the first time you ran the build. " +
+ "I wrote some sane defaults (for this machine) to 'gradle.properties', " +
+ "they will be picked up on consecutive gradle invocations (not this one).\n\n" +
+ "Run gradlew :helpLocalSettings for more information.")
+ }
+ }
+ }
+
+ // Disable any tasks in this build, they were forked recursively.
+ gradle.taskGraph.whenReady { graph ->
+ graph.allTasks.each { task ->
+ if (task != rootProject.setupLocalDefaultsOnce) {
+ task.enabled = false
+ }
+ }
+ }
+
+ // Make all tasks depend on local setup to make sure it'll run though.
+ allprojects {
+ tasks.all { task ->
+ if (task != rootProject.setupLocalDefaultsOnce) {
+ task.dependsOn rootProject.setupLocalDefaultsOnce
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/gradle/help.gradle b/gradle/help.gradle
index 0ee82ac..e5f8014 100644
--- a/gradle/help.gradle
+++ b/gradle/help.gradle
@@ -6,6 +6,7 @@ configure(rootProject) {
["Ant", "help/ant.txt", "Ant-gradle migration help."],
["Tests", "help/tests.txt", "Tests, filtering, beasting, etc."],
["ForbiddenApis", "help/forbiddenApis.txt", "How to add/apply rules for forbidden APIs."],
+ ["LocalSettings", "help/localSettings.txt", "Local settings, overrides and build performance tweaks."]
]
helpFiles.each { section, path, sectionInfo ->
@@ -24,7 +25,8 @@ configure(rootProject) {
println "This is an experimental Lucene/Solr gradle build. See some"
println "guidelines, ant-equivalent commands etc. under help/*; or type:"
helpFiles.each { section, path, sectionInfo ->
- println " gradlew :help${section} # ${sectionInfo}"
+ println String.format(Locale.ROOT,
+ " gradlew :help%-14s # %s", section, sectionInfo)
}
}
}
diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle
index 668cb84..d685328 100644
--- a/gradle/testing/defaults-tests.gradle
+++ b/gradle/testing/defaults-tests.gradle
@@ -13,8 +13,7 @@ allprojects {
useJUnit()
- // Set up default parallel execution limits.
- maxParallelForks = (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 3.0))
+ maxParallelForks = propertyOrDefault("tests.jvms", (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 4.0)))
minHeapSize = "256m"
maxHeapSize = "512m"
diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle
index 8024264..6330777 100644
--- a/gradle/testing/randomization.gradle
+++ b/gradle/testing/randomization.gradle
@@ -1,23 +1,6 @@
// Configure test randomization seeds and derived test properties.
-allprojects {
- ext {
- // Support passing overrides via -P or -D.
- propertyOrDefault = { propName, defValue ->
- def result
- if (project.hasProperty(propName)) {
- result = project.getProperty(propName)
- } else if (System.properties.containsKey(propName)) {
- result = System.properties.get(propName)
- } else {
- result = defValue
- }
- return result
- }
- }
-}
-
// Pick the "root" seed from which everything else is derived.
configure(rootProject) {
ext {
diff --git a/gradle/testing/slowest-tests-at-end.gradle b/gradle/testing/slowest-tests-at-end.gradle
index 2e33917..b3e6d20 100644
--- a/gradle/testing/slowest-tests-at-end.gradle
+++ b/gradle/testing/slowest-tests-at-end.gradle
@@ -8,7 +8,7 @@ allprojects {
def duration = (result.getEndTime() - result.getStartTime())
allTests << [
- name : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.name})",
+ name : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.path})",
duration: duration
]
}
diff --git a/help/localSettings.txt b/help/localSettings.txt
new file mode 100644
index 0000000..c67c895
--- /dev/null
+++ b/help/localSettings.txt
@@ -0,0 +1,46 @@
+Local developer settings
+========================
+
+The first invocation of any task in Lucene/Solr gradle build will generate
+and save a project-local 'gradle.properties' file. This file contains
+the defaults you may (but don't have to) tweak for your particular hardware
+(or taste).
+
+This is an overview of some of these settings.
+
+Parallelism
+-----------
+
+Gradle build can run tasks in parallel but by default it consumes all CPU cores which
+is too optimistic a default for Lucene/Solr tests. You can disable the parallelism
+entirely or assign it a 'low' priority with these properties:
+
+org.gradle.parallel=[true, false]
+org.gradle.priority=[normal, low]
+
+The default level of parallelism is computed based on the number of cores on
+your machine (on the first run of gradle build). By default these are fairly conservative
+settings (half the number of cores for workers, for example):
+
+org.gradle.workers.max=[X]
+tests.jvms=[N <= X]
+
+The number of test JVMs can be lower than the number of workers: this just means
+that two projects can run tests in parallel to saturate all the workers. The I/O and memory
+bandwidth limits will kick in quickly so even if you have a very beefy machine bumping
+it too high may not help.
+
+You can always override these settings locally using command line as well:
+gradlew -Ptests.jvms=N --max-workers=X
+
+Gradle Daemon
+-------------
+
+The gradle daemon is a background process that keeps an evaluated copy of the project
+structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't
+like the idea of having a (sizeable) background process running in the background,
+disable it.
+
+org.gradle.daemon=[true, false]
+org.gradle.jvmargs=...
+