You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2022/02/08 19:05:41 UTC
[kafka] branch trunk updated: MINOR: Reduce scala compilation time by 15% via scalac backend parallelism (#11739)
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new ca375d8 MINOR: Reduce scala compilation time by 15% via scalac backend parallelism (#11739)
ca375d8 is described below
commit ca375d8004c1ef070394d29028744d68b6feebd0
Author: Ismael Juma <is...@juma.me.uk>
AuthorDate: Tue Feb 8 11:03:19 2022 -0800
MINOR: Reduce scala compilation time by 15% via scalac backend parallelism (#11739)
Introduce `maxScalacThreads` and set the default to the lowest of `8`
and the number of processors available to the JVM. The number `8` was
picked empirically, the sweet spot is between 6 and 10.
On my desktop, `./gradlew clean core:compileScala core:compileTestScala`
improved from around 60s to 51s ( with this change.
While at it, we improve the build output to include more useful
information at the start: build id, max parallel forks, max scala
threads and max test retries.
Reviewers: Manikumar Reddy <ma...@gmail.com>, Sean Li
---
README.md | 6 ++++--
build.gradle | 67 ++++++++++++++++++++++++++++++++----------------------------
2 files changed, 40 insertions(+), 33 deletions(-)
diff --git a/README.md b/README.md
index f011c61..1f35df4 100644
--- a/README.md
+++ b/README.md
@@ -230,13 +230,15 @@ The following options should be set with a `-P` switch, for example `./gradlew -
* `commitId`: sets the build commit ID as .git/HEAD might not be correct if there are local commits added for build purposes.
* `mavenUrl`: sets the URL of the maven deployment repository (`file://path/to/repo` can be used to point to a local repository).
-* `maxParallelForks`: limits the maximum number of processes for each task.
+* `maxParallelForks`: maximum number of test processes to start in parallel. Defaults to the number of processors available to the JVM.
+* `maxScalacThreads`: maximum number of worker threads for the scalac backend. Defaults to the lowest of `8` and the number of processors
+available to the JVM. The value must be between 1 and 16 (inclusive).
* `ignoreFailures`: ignore test failures from junit
* `showStandardStreams`: shows standard out and standard error of the test JVM(s) on the console.
* `skipSigning`: skips signing of artifacts.
* `testLoggingEvents`: unit test events to be logged, separated by comma. For example `./gradlew -PtestLoggingEvents=started,passed,skipped,failed test`.
* `xmlSpotBugsReport`: enable XML reports for spotBugs. This also disables HTML reports as only one can be enabled at a time.
-* `maxTestRetries`: the maximum number of retries for a failing test case.
+* `maxTestRetries`: maximum number of retries for a failing test case.
* `maxTestRetryFailures`: maximum number of test failures before retrying is disabled for subsequent tests.
* `enableTestCoverage`: enables test coverage plugins and tasks, including bytecode enhancement of classes required to track said
coverage. Note that this introduces some overhead when running tests and hence why it's disabled by default (the overhead
diff --git a/build.gradle b/build.gradle
index b7b184c..d7ed256 100644
--- a/build.gradle
+++ b/build.gradle
@@ -119,7 +119,9 @@ ext {
"--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED"
)
- userMaxForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : null
+ maxTestForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : Runtime.runtime.availableProcessors()
+ maxScalacThreads = project.hasProperty('maxScalacThreads') ? maxScalacParallelism.toInteger() :
+ Math.min(Runtime.runtime.availableProcessors(), 8)
userIgnoreFailures = project.hasProperty('ignoreFailures') ? ignoreFailures : false
userMaxTestRetries = project.hasProperty('maxTestRetries') ? maxTestRetries.toInteger() : 0
@@ -146,7 +148,26 @@ ext {
generatedDocsDir = new File("${project.rootDir}/docs/generated")
- commitId = project.hasProperty('commitId') ? commitId : null
+ commitId = determineCommitId()
+}
+
+def determineCommitId() {
+ def takeFromHash = 16
+ if (project.hasProperty('commitId2')) {
+ commitId2.take(takeFromHash)
+ } else if (file("$rootDir/.git/HEAD").exists()) {
+ def headRef = file("$rootDir/.git/HEAD").text
+ if (headRef.contains('ref: ')) {
+ headRef = headRef.replaceAll('ref: ', '').trim()
+ if (file("$rootDir/.git/$headRef").exists()) {
+ file("$rootDir/.git/$headRef").text.trim().take(takeFromHash)
+ }
+ } else {
+ headRef.trim().take(takeFromHash)
+ }
+ } else {
+ "unknown"
+ }
}
apply from: file('wrapper.gradle')
@@ -186,7 +207,8 @@ if (file('.git').exists()) {
} else {
rat.enabled = false
}
-println("Starting build with version $version using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
+println("Starting build with version $version (commit id ${commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
+println("Build properties: maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")
subprojects {
@@ -391,7 +413,7 @@ subprojects {
}
test {
- maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
+ maxParallelForks = maxTestForks
ignoreFailures = userIgnoreFailures
maxHeapSize = defaultMaxHeapSize
@@ -416,7 +438,7 @@ subprojects {
}
task integrationTest(type: Test, dependsOn: compileJava) {
- maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
+ maxParallelForks = maxTestForks
ignoreFailures = userIgnoreFailures
maxHeapSize = defaultMaxHeapSize
@@ -449,7 +471,7 @@ subprojects {
}
task unitTest(type: Test, dependsOn: compileJava) {
- maxParallelForks = userMaxForks ?: Runtime.runtime.availableProcessors()
+ maxParallelForks = maxTestForks
ignoreFailures = userIgnoreFailures
maxHeapSize = defaultMaxHeapSize
@@ -561,6 +583,7 @@ subprojects {
}
tasks.withType(ScalaCompile) {
+
scalaCompileOptions.additionalParameters = [
"-deprecation",
"-unchecked",
@@ -570,6 +593,7 @@ subprojects {
"-language:postfixOps",
"-language:implicitConversions",
"-language:existentials",
+ "-Ybackend-parallelism", maxScalacThreads.toString(),
"-Xlint:constant",
"-Xlint:delayedinit-select",
"-Xlint:doc-detached",
@@ -689,25 +713,6 @@ subprojects {
task reportCoverage(dependsOn: [coverageGen])
}
- task determineCommitId {
- def takeFromHash = 16
- if (commitId) {
- commitId = commitId.take(takeFromHash)
- } else if (file("$rootDir/.git/HEAD").exists()) {
- def headRef = file("$rootDir/.git/HEAD").text
- if (headRef.contains('ref: ')) {
- headRef = headRef.replaceAll('ref: ', '').trim()
- if (file("$rootDir/.git/$headRef").exists()) {
- commitId = file("$rootDir/.git/$headRef").text.trim().take(takeFromHash)
- }
- } else {
- commitId = headRef.trim().take(takeFromHash)
- }
- } else {
- commitId = "unknown"
- }
- }
-
}
gradle.taskGraph.whenReady { taskGraph ->
@@ -1214,7 +1219,7 @@ project(':clients') {
testImplementation libs.jacksonJaxrsJsonProvider
}
- task createVersionFile(dependsOn: determineCommitId) {
+ task createVersionFile() {
ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
outputs.file receiptFile
outputs.upToDateWhen { false }
@@ -1329,7 +1334,7 @@ project(':raft') {
testRuntimeOnly libs.slf4jlog4j
}
- task createVersionFile(dependsOn: determineCommitId) {
+ task createVersionFile() {
ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
outputs.file receiptFile
outputs.upToDateWhen { false }
@@ -1408,7 +1413,7 @@ project(':server-common') {
testRuntimeOnly libs.slf4jlog4j
}
- task createVersionFile(dependsOn: determineCommitId) {
+ task createVersionFile() {
ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
outputs.file receiptFile
outputs.upToDateWhen { false }
@@ -1464,7 +1469,7 @@ project(':storage:api') {
testRuntimeOnly libs.slf4jlog4j
}
- task createVersionFile(dependsOn: determineCommitId) {
+ task createVersionFile() {
ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
outputs.file receiptFile
outputs.upToDateWhen { false }
@@ -1530,7 +1535,7 @@ project(':storage') {
testRuntimeOnly libs.slf4jlog4j
}
- task createVersionFile(dependsOn: determineCommitId) {
+ task createVersionFile() {
ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName")
outputs.file receiptFile
outputs.upToDateWhen { false }
@@ -1806,7 +1811,7 @@ project(':streams') {
duplicatesStrategy 'exclude'
}
- task createStreamsVersionFile(dependsOn: determineCommitId) {
+ task createStreamsVersionFile() {
ext.receiptFile = file("$buildDir/kafka/$buildStreamsVersionFileName")
outputs.file receiptFile
outputs.upToDateWhen { false }