You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/14 20:07:09 UTC
(pulsar) branch master updated: [improve][build] Add validated API/SPI publication subset (#26579)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 3e9fe6415c11 [improve][build] Add validated API/SPI publication subset (#26579)
3e9fe6415c11 is described below
commit 3e9fe6415c11a669449640621826a7f2c62e0991
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Mon Sep 14 23:07:03 2026 +0300
[improve][build] Add validated API/SPI publication subset (#26579)
---
.github/workflows/pulsar-ci.yaml | 6 +
CONTRIBUTING.md | 5 +
build-logic/PUBLISHING.md | 285 +++++++++++++++++++++
build-logic/conventions/build.gradle.kts | 5 +
.../src/main/kotlin/PulsarApiSpiPublication.kt | 76 ++++++
.../src/main/kotlin/ValidateApiSpiPublication.kt | 109 ++++++++
.../main/kotlin/pulsar.nar-conventions.gradle.kts | 5 +-
...lsar.public-java-library-conventions.gradle.kts | 3 +-
.../kotlin/pulsar.publish-conventions.gradle.kts | 9 +-
...sar.publish-repositories-conventions.gradle.kts | 31 +++
.../test/kotlin/ValidateApiSpiPublicationTest.kt | 98 +++++++
buildtools/build.gradle.kts | 5 +
pulsar-bom/build.gradle.kts | 103 ++++----
pulsar-client/build.gradle.kts | 2 +-
pulsar-functions/localrun-shaded/build.gradle.kts | 3 +-
15 files changed, 691 insertions(+), 54 deletions(-)
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index a79842c54076..9d16d3b0440c 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -200,6 +200,12 @@ jobs:
- name: Check that project's public libraries can be published to a Maven repository
run: ./gradlew publishAllPublicationsToLocalDeployRepository
+ - name: Check API/SPI publication dependency closure with a custom group
+ # Validate generated POMs and Gradle metadata, including the BOM, without uploading artifacts.
+ run: >-
+ ./gradlew validateApiSpiPublication
+ -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar
+
- name: Upload Gradle reports
uses: actions/upload-artifact@v4
if: ${{ !success() }}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ec3f0a211526..b1a24db8d922 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -63,6 +63,11 @@ For the Gradle build infrastructure and how to change build files (convention pl
catalog, configuration-cache rules), see
[`ARCHITECTURE.md` → Build infrastructure](ARCHITECTURE.md#build-infrastructure).
+### Publishing artifacts
+
+See [Publishing Maven artifacts](build-logic/PUBLISHING.md) for ordinary publishing, the
+API/SPI subset, custom repositories, groups and credentials.
+
## Running tests
Most of these per-module "unit tests" are actually **integration-style** — they start a real in-JVM
diff --git a/build-logic/PUBLISHING.md b/build-logic/PUBLISHING.md
new file mode 100644
index 000000000000..ae80240571e1
--- /dev/null
+++ b/build-logic/PUBLISHING.md
@@ -0,0 +1,285 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Publishing Maven artifacts
+
+Run the commands below from the repository root using `./gradlew`. The shared publication and
+repository conventions live in [`conventions/src/main/kotlin`](conventions/src/main/kotlin).
+
+## Ordinary publishing
+
+By default (`publishApiAndSpiOnly` unset or `false`), Gradle publishes the modules that apply the
+publishing conventions, including the root `pulsar` parent POM and the full `pulsar-bom`. The
+repository's `gradle.properties` supplies the default group (`org.apache.pulsar`) and version.
+
+Choose the task for the intended destination:
+
+| Task | Destination | Version requirement |
+| --- | --- | --- |
+| `publishAllPublicationsToLocalDeployRepository` | `build/local-deploy-repo` | Any version |
+| `publishToMavenLocal` | Local Maven repository, normally `~/.m2/repository` | Any version |
+| `publishAllPublicationsToApacheSnapshotsRepository` | ASF Nexus snapshots | Ends in `-SNAPSHOT` |
+| `publishAllPublicationsToApacheReleasesRepository` | ASF Nexus staging | Does not end in `-SNAPSHOT` |
+
+The remote publishing examples below assume a non-`SNAPSHOT` version. For snapshots, use the
+alternative task noted in each example.
+
+```bash
+# Inspect the complete publication set in a local directory.
+./gradlew publishAllPublicationsToLocalDeployRepository
+
+# Publish a release using credentials supplied by the environment or properties file.
+# For snapshots, use publishAllPublicationsToApacheSnapshotsRepository instead.
+./gradlew publishAllPublicationsToApacheReleasesRepository
+```
+
+Use an unqualified task name from the root to publish across projects. A fully qualified task such
+as `:pulsar-client-api:publishMavenPublicationToApacheReleasesRepository` publishes just that
+project; it does not publish its dependency artifacts. Avoid the generic `publish` task when you
+intend to target only one repository: it targets all configured publishing repositories.
+
+### ASF release staging and signing
+
+For an Apache Pulsar release, follow the canonical
+[release process: stage artifacts in ASF Nexus](https://github.com/apache/pulsar-site/blob/main/contribute/release-process.md#stage-artifacts-in-the-asf-nexus-repository)
+for release preparation, signing-key setup, staging, and closing the staging repository. From a
+prepared release checkout with a non-snapshot version, the Gradle upload command is:
+
+```bash
+ # ASF_USERNAME, ASF_PASSWORD and APACHE_USER_GPGID are supplied by your environment.
+ # For snapshots, use publishAllPublicationsToApacheSnapshotsRepository
+ # and the apacheSnapshotsUsername/apacheSnapshotsPassword properties.
+ ORG_GRADLE_PROJECT_apacheReleasesUsername="$ASF_USERNAME" \
+ORG_GRADLE_PROJECT_apacheReleasesPassword="$ASF_PASSWORD" \
+./gradlew publishAllPublicationsToApacheReleasesRepository \
+ -PuseGpgCmd=true -Psigning.gnupg.keyName="$APACHE_USER_GPGID"
+```
+
+`-PuseGpgCmd=true` enables command-line GPG signing. `signing.gnupg.keyName` selects a key; if
+omitted, GPG uses its default key. Signing tasks are disabled when no signing configuration is
+present, which supports local development. Configure signing for ASF release publication.
+The build serializes Maven uploads within one Gradle invocation, so `--no-parallel` is unnecessary.
+For ASF staging, finish and close one staging repository before starting another release upload.
+
+## Custom repositories, groups and credentials
+
+The existing repository definitions can target a custom Maven repository without editing build
+scripts. Override these Gradle properties:
+
+| Repository | URL property | Credential properties |
+| --- | --- | --- |
+| `apacheSnapshots` | `apacheSnapshotsRepoUrl` | `apacheSnapshotsUsername`, `apacheSnapshotsPassword` |
+| `apacheReleases` | `apacheReleasesRepoUrl` | `apacheReleasesUsername`, `apacheReleasesPassword` |
+
+The task names and snapshot/release version checks stay the same even when the URLs are overridden.
+For example, a GitHub Packages destination still uses `publishAllPublicationsToApacheSnapshotsRepository`
+for snapshots and `publishAllPublicationsToApacheReleasesRepository` for release versions.
+
+Set `group` to publish under a different Maven group. It applies to the parent POM, module
+coordinates, BOM entries and publication-only Pulsar dependency replacements. Changing the group
+does not change Java package names or imports, so switching to the custom group requires dependency
+configuration changes only; no application source-code changes are required because of the group
+change. The group override works with either ordinary publishing or the API/SPI subset.
+Set a custom build version with
+`-Pversion=5.0.0+mypatch.1`; it applies to all published artifacts, including the parent POM and BOM.
+Use a version ending in `-SNAPSHOT` when publishing to the snapshots repository.
+
+### Properties file in ephemeral CI
+
+On an ephemeral CI worker, with `REPO_USERNAME` and `REPO_PASSWORD` supplied by CI secrets, the
+following configures both repository definitions in the Gradle user properties file. The publishing
+commands below set the custom group with `-Pgroup=myorg.pulsar`.
+This example assumes the default Gradle user home, `~/.gradle`; when `GRADLE_USER_HOME` is set, use
+`$GRADLE_USER_HOME/gradle.properties` instead.
+
+```bash
+mkdir -p ~/.gradle
+cat >> ~/.gradle/gradle.properties <<EOL
+apacheSnapshotsRepoUrl=https://maven.pkg.github.com/myorg/pulsar
+apacheSnapshotsUsername=${REPO_USERNAME}
+apacheSnapshotsPassword=${REPO_PASSWORD}
+apacheReleasesRepoUrl=https://maven.pkg.github.com/myorg/pulsar
+apacheReleasesUsername=${REPO_USERNAME}
+apacheReleasesPassword=${REPO_PASSWORD}
+EOL
+```
+
+The properties file contains the expanded credentials. Keep it private to the CI worker, outside
+version control, and do not print it in build logs. Values use Java properties syntax; credentials
+containing backslashes or line breaks need escaping. Environment variables avoid that encoding step.
+
+Publish a custom release:
+
+```bash
+./gradlew publishAllPublicationsToApacheReleasesRepository \
+ -Pgroup=myorg.pulsar -Pversion=5.0.0+mypatch.1
+```
+
+Publish a custom snapshot:
+
+```bash
+./gradlew publishAllPublicationsToApacheSnapshotsRepository \
+ -Pgroup=myorg.pulsar -Pversion=5.0.0+mypatch.1-SNAPSHOT
+```
+
+### Environment variables and command-line overrides
+
+Prefix a property name with `ORG_GRADLE_PROJECT_` to supply it through the environment:
+
+```bash
+ # For snapshots, use publishAllPublicationsToApacheSnapshotsRepository and a -SNAPSHOT version,
+ # and replace apacheReleases with apacheSnapshots in the environment variable names.
+ ORG_GRADLE_PROJECT_apacheReleasesRepoUrl=https://maven.pkg.github.com/myorg/pulsar \
+ORG_GRADLE_PROJECT_apacheReleasesUsername="$REPO_USERNAME" \
+ORG_GRADLE_PROJECT_apacheReleasesPassword="$REPO_PASSWORD" \
+./gradlew publishAllPublicationsToApacheReleasesRepository \
+ -Pgroup=myorg.pulsar -Pversion=5.0.0+mypatch.1
+```
+
+The credential command blocks intentionally begin with a space for use in interactive shells.
+To omit commands beginning with a space from history, enable `HISTCONTROL=ignorespace` (or
+`ignoreboth`) in [Bash](https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html), or
+`setopt HIST_IGNORE_SPACE` in [Zsh](https://zsh.sourceforge.io/Doc/Release/Options.html#History).
+Many shell configurations already enable this behavior; check yours before entering credentials.
+Always prefix interactive password assignments with a space, including a separate `export` or
+assignment used to prepare `REPO_PASSWORD` or `ASF_PASSWORD`. If a block begins with a comment,
+keep the space before `#` and also before the first line of the actual command. Continuation lines
+following `\` do not need an additional leading space. This history guidance applies to interactive
+shells; ordinary non-interactive CI scripts do not record shell history.
+
+All these properties also accept `-Pname=value` on the Gradle command line. Prefer a private
+properties file or environment variables for passwords: `-P...Password=...` exposes the value as a
+process argument and can record it in shell history. Keep shell tracing disabled when handling
+credentials in CI.
+
+Command-line `-P` values override properties files. User-level `gradle.properties` overrides the
+repository's properties file, and properties files override `ORG_GRADLE_PROJECT_` variables.
+Consequently, `ORG_GRADLE_PROJECT_group` alone does not override this repository's existing `group`
+property; use the user properties file or `-Pgroup=myorg.pulsar`. See
+[Gradle project properties](https://docs.gradle.org/current/userguide/build_environment.html#sec:project_properties)
+for the full precedence rules.
+
+### Using the custom artifacts in a Gradle or Maven build
+
+Configure the custom Maven repository, including any read credentials, in the consuming build's
+dependency repositories. The publishing URL properties above configure upload destinations only.
+In a Gradle consumer, add this dependency substitution rule to `build.gradle.kts` to replace the
+`org.apache.pulsar` group with `myorg.pulsar` for direct and transitive dependencies:
+
+```kotlin
+import org.gradle.api.artifacts.component.ModuleComponentSelector
+
+configurations.configureEach {
+ val customPulsarVersion = "5.0.0+mypatch.1"
+ resolutionStrategy.dependencySubstitution.all {
+ val module = requested
+ if (module is ModuleComponentSelector && module.group == "org.apache.pulsar") {
+ useTarget(
+ "myorg.pulsar:${module.module}:$customPulsarVersion",
+ "Use the custom Pulsar API/SPI build"
+ )
+ }
+ }
+}
+```
+
+The rule retains each artifact ID and selects the custom version. Apply it to each consuming
+project, for example through a shared convention plugin. Every requested Pulsar artifact must be
+available in the custom repository; when publishing only the API/SPI subset, dependencies on
+artifacts outside that subset need a narrower substitution rule or additional publications.
+See [Gradle dependency substitution](https://docs.gradle.org/current/userguide/resolution_rules.html#sec:dependency_substitution_rules).
+
+With Maven builds, switching to a custom group for the Pulsar API/SPI artifacts is more complicated.
+Exclude the original `org.apache.pulsar` artifacts explicitly from each dependency that introduces
+them, and add explicit dependencies on the replacement `myorg.pulsar` artifacts and custom version.
+Maven dependency management can manage versions, but changing the group creates different artifact
+coordinates; it does not redirect dependencies on the original group. Maven exclusions apply to
+individual dependency subtrees, so account for every path that brings in the original artifacts.
+See [Maven dependency exclusions](https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html#dependency-exclusions).
+
+## Publishing the API/SPI subset
+
+API/SPI publication provides the Maven dependencies needed by applications using the Pulsar Java
+client, Pulsar Functions, and Pulsar plugins such as interceptors and additional servlets. The set
+includes the client implementations (including v5), API and service provider interface (SPI)
+libraries, and their transitive Pulsar dependencies.
+
+A typical use case is distributing a custom Pulsar Java client build to your own Maven repository
+so applications can quickly adopt a bug fix or security fix in Pulsar itself. Publishing this subset
+avoids performing a full Pulsar dependency release to that repository. For deploying a custom server
+build, the Docker build is sufficient; API/SPI publication supplies the artifacts consumed by
+applications, Functions and plugins outside that server image.
+
+> **Note:** This custom-build use case is not intended for handling CVEs in transitive third-party
+> client dependencies. With Pulsar 5, the recommended approach for dependencies bundled inside
+> shaded JARs is to switch to **unshaded client dependencies** and update the affected dependency
+> through the consuming build's dependency management. There is no need to create a custom Pulsar
+> build just to replace a transitive dependency when using unshaded JARs. See the
+> [`pulsar-client-v5-all` guide](../pulsar-client-v5-all/README.md) for the unshaded aggregate and
+> migration instructions, including the exclusions needed to remove old shaded artifacts. This
+> also applies to applications using the ordinary v4 client API: the aggregate includes v4, v5
+> and admin implementations, so switching dependencies does not require migrating source code to
+> the v5 API. Verify compatibility with the replacement dependency.
+
+Use `-PpublishApiAndSpiOnly=true` to select this publication mode. The explicit project selection lives
+in [`PulsarApiSpiPublication.projects`](conventions/src/main/kotlin/PulsarApiSpiPublication.kt).
+Add a project's Gradle path there to include it. Without the property, publishing uses the normal
+release publication set.
+
+Override the group with `-Pgroup=<your.group>`. The parent POM and publication-only dependency
+replacements use the same group. In API/SPI mode, `pulsar-bom` retains only constraints for selected
+projects, in both its Maven POM and Gradle Module Metadata.
+
+```bash
+# Validate the selected artifacts' generated POMs and Gradle Module Metadata, without uploading.
+./gradlew validateApiSpiPublication -PpublishApiAndSpiOnly=true -Pgroup=com.example.pulsar
+
+# Publish the complete selection to build/local-deploy-repo for inspection.
+./gradlew publishAllPublicationsToLocalDeployRepository \
+ -PpublishApiAndSpiOnly=true -Pgroup=com.example.pulsar
+```
+
+Use the unqualified `publishAllPublicationsTo<Repository>Repository` task from the repository root
+for any configured Maven repository. Gradle selects that task across projects; projects outside the
+selection have no publications in this mode. A fully qualified per-project task publishes only that
+project, so use the unqualified command when publishing the complete set to a fresh repository.
+
+To publish the subset to a custom releases repository configured as above:
+
+```bash
+# For snapshots, use publishAllPublicationsToApacheSnapshotsRepository and a -SNAPSHOT version.
+./gradlew publishAllPublicationsToApacheReleasesRepository \
+ -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar -Pversion=5.0.0+mypatch.1
+```
+
+Every Maven upload (including `publishToMavenLocal`) in this mode depends on
+`validateApiSpiPublication`. Validation checks the generated metadata of **every selected project**,
+including parent references, platform constraints and publication-only dependency replacements.
+Every Pulsar coordinate must identify another selected publication at the same published version;
+checking all such edges establishes transitive closure. Missing projects fail validation before any
+upload, with the referring artifact and missing coordinate. Test/build dependencies and dependencies
+bundled inside shaded artifacts do not need separate publication unless the generated metadata
+references them. Generating Gradle metadata can build JARs, so this is not a source-only check.
+
+Both configure-on-demand and configuration cache remain supported. A scoped build such as
+`:pulsar-client-api:assemble` configures only the projects it needs even with API/SPI mode enabled.
+Validation and publication intentionally configure the entire selected publication set and the
+build dependencies needed to generate its metadata.
diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts
index 46120a3102ed..bd294d030d3e 100644
--- a/build-logic/conventions/build.gradle.kts
+++ b/build-logic/conventions/build.gradle.kts
@@ -22,6 +22,7 @@ plugins {
}
dependencies {
+ testImplementation(libs.testng)
// The Shadow plugin brings its own log4j-core onto the build classpath; align it with the
// log4j version the rest of the build uses (`log4j2` in the version catalog).
implementation(platform(libs.log4j.bom))
@@ -35,3 +36,7 @@ dependencies {
"${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}"
})
}
+
+tasks.test {
+ useTestNG()
+}
diff --git a/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt b/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt
new file mode 100644
index 000000000000..63f2420102a6
--- /dev/null
+++ b/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Project
+
+/** API/SPI publication set for Java clients, Functions and plugins, including their published dependencies. */
+object PulsarApiSpiPublication {
+ val projects: Set<String> = setOf(
+ ":",
+ ":buildtools",
+ ":managed-ledger",
+ ":pulsar-bom",
+ ":pulsar-broker",
+ ":pulsar-broker-auth-sasl",
+ ":pulsar-broker-common",
+ ":pulsar-cli-utils",
+ ":pulsar-client-admin-api",
+ ":pulsar-client-admin-original",
+ ":pulsar-client-admin-shaded",
+ ":pulsar-client-all",
+ ":pulsar-client-api",
+ ":pulsar-client-api-v5",
+ ":pulsar-client-auth-sasl",
+ ":pulsar-client-fastutil-minimized",
+ ":pulsar-client-messagecrypto-bc",
+ ":pulsar-client-original",
+ ":pulsar-client-shaded",
+ ":pulsar-client-v5",
+ ":pulsar-client-v5-all",
+ ":pulsar-client-v5-shaded",
+ ":pulsar-common",
+ ":pulsar-config-validation",
+ ":pulsar-dependencies",
+ ":pulsar-docs-tools",
+ ":pulsar-functions:pulsar-functions-api",
+ ":pulsar-functions:pulsar-functions-instance",
+ ":pulsar-functions:pulsar-functions-proto",
+ ":pulsar-functions:pulsar-functions-runtime",
+ ":pulsar-functions:pulsar-functions-secrets",
+ ":pulsar-functions:pulsar-functions-utils",
+ ":pulsar-functions:pulsar-functions-worker",
+ ":pulsar-http-client-api",
+ ":pulsar-io:pulsar-io-core",
+ ":pulsar-metadata",
+ ":pulsar-opentelemetry",
+ ":pulsar-package-management:pulsar-package-core",
+ ":pulsar-package-management:pulsar-package-filesystem-storage",
+ ":pulsar-proxy",
+ ":pulsar-tls-factory-api",
+ ":pulsar-transaction:pulsar-transaction-common",
+ ":pulsar-transaction:pulsar-transaction-coordinator",
+ ":pulsar-websocket",
+ ":testmocks",
+ )
+
+ fun isEnabled(project: Project): Boolean =
+ project.providers.gradleProperty("publishApiAndSpiOnly").getOrElse("false").toBoolean()
+
+ fun includes(project: Project): Boolean = !isEnabled(project) || project.path in projects
+}
diff --git a/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt b/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt
new file mode 100644
index 000000000000..e9ae0714081e
--- /dev/null
+++ b/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import groovy.json.JsonSlurper
+import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.TaskAction
+import org.gradle.work.DisableCachingByDefault
+import org.w3c.dom.Element
+import javax.xml.parsers.DocumentBuilderFactory
+
+/** Checks the consumer graph, including publication-only rewrites and dependency-reduced shaded POMs. */
+@DisableCachingByDefault(because = "Verification has no outputs")
+abstract class ValidateApiSpiPublication : DefaultTask() {
+ @get:InputFiles
+ @get:PathSensitive(PathSensitivity.RELATIVE)
+ abstract val poms: ConfigurableFileCollection
+
+ @get:InputFiles
+ @get:PathSensitive(PathSensitivity.RELATIVE)
+ abstract val moduleMetadata: ConfigurableFileCollection
+
+ @get:Input
+ abstract val publicationGroup: Property<String>
+
+ @TaskAction
+ fun validate() {
+ val factory = DocumentBuilderFactory.newInstance()
+ factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
+ fun Element.childText(name: String): String = (0 until childNodes.length)
+ .map { childNodes.item(it) }.filterIsInstance<Element>()
+ .firstOrNull { it.tagName == name }?.textContent.orEmpty()
+ val documents = poms.files.sorted().associateWith {
+ factory.newDocumentBuilder().parse(it).documentElement
+ }
+ val coordinates = documents.values.map {
+ "${it.childText("groupId")}:${it.childText("artifactId")}:${it.childText("version")}"
+ }.toSet()
+ val failures = sortedSetOf<String>()
+ fun check(source: String, group: String, artifact: String, version: String) {
+ if (group == publicationGroup.get() || group == "org.apache.pulsar") {
+ val coordinate = "$group:$artifact:$version"
+ if (coordinate !in coordinates) {
+ failures.add("$source -> $coordinate")
+ }
+ }
+ }
+ for ((file, root) in documents) {
+ val source = root.childText("artifactId") + " (" + file.name + ")"
+ if (root.childText("groupId") != publicationGroup.get()) {
+ failures.add("$source publishes under ${root.childText("groupId")} instead of ${publicationGroup.get()}")
+ }
+ for (tag in listOf("parent", "dependency")) {
+ val nodes = root.getElementsByTagName(tag)
+ for (i in 0 until nodes.length) {
+ val node = nodes.item(i) as Element
+ check(source, node.childText("groupId"), node.childText("artifactId"), node.childText("version"))
+ }
+ }
+ }
+ for (file in moduleMetadata.files.sorted()) {
+ val json = JsonSlurper().parse(file) as Map<*, *>
+ val component = json["component"] as Map<*, *>
+ val source = "${component["module"]} (Gradle metadata)"
+ for (variant in json["variants"] as List<*>) {
+ val data = variant as Map<*, *>
+ for (key in listOf("dependencies", "dependencyConstraints")) {
+ for (entry in data[key] as? List<*> ?: emptyList<Any>()) {
+ val dep = entry as Map<*, *>
+ val version = dep["version"] as? Map<*, *>
+ check(source, dep["group"].toString(), dep["module"].toString(),
+ (version?.get("strictly") ?: version?.get("requires") ?: version?.get("prefers")).toString())
+ }
+ }
+ (data["available-at"] as? Map<*, *>)?.let {
+ check(source, it["group"].toString(), it["module"].toString(), it["version"].toString())
+ }
+ }
+ }
+ if (failures.isNotEmpty()) {
+ throw GradleException("API/SPI publication contains unpublished Pulsar dependencies:\n" +
+ failures.joinToString("\n") +
+ "\nAdd the missing projects to PulsarApiSpiPublication.projects or correct their published coordinates.")
+ }
+ logger.lifecycle("Validated {} API/SPI publications and their published dependency closure.", coordinates.size)
+ }
+}
diff --git a/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts
index c0c9c1b7eb42..c71df08bc18b 100644
--- a/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts
+++ b/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts
@@ -50,13 +50,14 @@ val pulsarPlatformModules = setOf(
"pulsar-package-core",
)
+val pulsarGroup = project.group.toString()
configurations.named("runtimeClasspath") {
exclude(group = "org.apache.bookkeeper")
// Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it.
// Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts.
exclude(group = "com.google.protobuf")
pulsarPlatformModules.forEach { module ->
- exclude(group = "org.apache.pulsar", module = module)
+ exclude(group = pulsarGroup, module = module)
}
}
@@ -97,7 +98,7 @@ if (parentProject != null && parentProject != rootProject && parentProject.paren
// NAR modules bundle all dependencies, so the POM should have no <dependencies> section.
publishing {
publications {
- named<MavenPublication>("maven") {
+ withType<MavenPublication>().configureEach {
// Replace component-based artifacts with just the NAR file
artifacts.clear()
artifact(tasks.named("nar"))
diff --git a/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts
index 5841e7d70baa..ac3cac573f96 100644
--- a/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts
+++ b/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts
@@ -30,7 +30,8 @@ plugins {
// in scopes that end up in the published POM (api, implementation, runtimeOnly).
// Test/compileOnly scoped dependencies are excluded since they don't appear in the POM.
// NAR modules are not validated here — they bundle all dependencies and have empty POMs.
-run {
+// API/SPI mode validates the generated consumer metadata for the entire selection before uploading.
+if (!PulsarApiSpiPublication.isEnabled(project)) {
val publishedScopes = listOf("api", "implementation", "runtimeOnly", "shadow", "shadowApi")
val configsToCheck = publishedScopes.mapNotNull { name ->
configurations.findByName(name)?.let { name to it }
diff --git a/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts
index 58b11e016127..253fd5b6ee33 100644
--- a/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts
+++ b/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts
@@ -28,6 +28,9 @@ plugins {
// --- java-library projects: JAR + sources + javadoc ---
pluginManager.withPlugin("java-library") {
+ if (!PulsarApiSpiPublication.includes(project)) {
+ return@withPlugin
+ }
val sourceSets = the<SourceSetContainer>()
// Match Maven's javadoc configuration: no doclint, don't fail on errors
@@ -90,6 +93,9 @@ pluginManager.withPlugin("java-library") {
// --- java-platform projects (BOM, dependencies): POM-only, no JAR ---
pluginManager.withPlugin("java-platform") {
+ if (!PulsarApiSpiPublication.includes(project)) {
+ return@withPlugin
+ }
publishing {
publications {
create<MavenPublication>("maven") {
@@ -107,6 +113,7 @@ run {
val isPlatformProject = plugins.hasPlugin("java-platform")
val isRootProject = project == rootProject
val pulsarVersion = version.toString()
+ val pulsarGroup = project.group.toString()
// Per-module POM name and description. Read in afterEvaluate so that a description
// assigned in a module's build script body is picked up, and captured as plain strings
@@ -162,7 +169,7 @@ run {
s = s.replace(
"<modelVersion>4.0.0</modelVersion>",
"<modelVersion>4.0.0</modelVersion>\n <parent>\n" +
- " <groupId>org.apache.pulsar</groupId>\n" +
+ " <groupId>$pulsarGroup</groupId>\n" +
" <artifactId>pulsar</artifactId>\n" +
" <version>$pulsarVersion</version>\n" +
" </parent>"
diff --git a/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts
index 4fe98e0e1ad0..2ace22ea59a2 100644
--- a/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts
+++ b/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts
@@ -167,3 +167,34 @@ tasks.withType<Sign>().configureEach {
providers.gradleProperty("signing.gnupg.keyName").isPresent ||
(providers.gradleProperty("useGpgCmd").orNull?.toBoolean() ?: false)
}
+
+// Every upload in API/SPI mode waits for the entire selected consumer graph to pass validation.
+// String task dependencies also configure the selected projects with configure-on-demand enabled.
+if (PulsarApiSpiPublication.isEnabled(project)) {
+ if (project == rootProject) {
+ tasks.register<ValidateApiSpiPublication>("validateApiSpiPublication") {
+ group = "verification"
+ description = "Validate that every published Pulsar dependency belongs to the API/SPI publication set."
+ publicationGroup.set(project.group.toString())
+ for (selectedPath in PulsarApiSpiPublication.projects) {
+ val selected = project(selectedPath)
+ // A qualified task invocation does not discover/configure container projects.
+ // Initialize their Kotlin script scopes before Gradle resolves nested task paths.
+ generateSequence(selected.parent) { it.parent }
+ .takeWhile { it != rootProject }.toList().asReversed().forEach {
+ project.evaluationDependsOn(it.path)
+ }
+ val prefix = if (selectedPath == ":") "" else selectedPath
+ poms.from(selected.layout.buildDirectory.file("publications/maven/pom-default.xml"))
+ dependsOn("$prefix:generatePomFileForMavenPublication")
+ if (selectedPath != ":") {
+ moduleMetadata.from(selected.layout.buildDirectory.file("publications/maven/module.json"))
+ dependsOn("$prefix:generateMetadataFileForMavenPublication")
+ }
+ }
+ }
+ }
+ tasks.withType<AbstractPublishToMaven>().configureEach {
+ dependsOn(":validateApiSpiPublication")
+ }
+}
diff --git a/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt b/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt
new file mode 100644
index 000000000000..cda695d6f821
--- /dev/null
+++ b/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.GradleException
+import org.gradle.testfixtures.ProjectBuilder
+import org.testng.Assert.assertTrue
+import org.testng.Assert.expectThrows
+import org.testng.annotations.Test
+import java.nio.file.Files
+
+class ValidateApiSpiPublicationTest {
+ private fun verify(pomDependency: String = "", metadataDependency: String = "", parent: String = "") {
+ val directory = Files.createTempDirectory("api-spi-publication-test").toFile()
+ try {
+ val project = ProjectBuilder.builder().withProjectDir(directory).build()
+ val task = project.tasks.register("validate", ValidateApiSpiPublication::class.java).get()
+ task.publicationGroup.set("example.pulsar")
+ val pom = directory.resolve("pom.xml")
+ pom.writeText("""
+ <project><modelVersion>4.0.0</modelVersion>
+ $parent
+ <groupId>example.pulsar</groupId><artifactId>client</artifactId><version>1</version>
+ <dependencies>$pomDependency</dependencies></project>
+ """.trimIndent())
+ val module = directory.resolve("module.json")
+ module.writeText("""
+ {"component":{"group":"example.pulsar","module":"client","version":"1"},
+ "variants":[{"name":"runtime","dependencies":[$metadataDependency]}]}
+ """.trimIndent())
+ task.poms.from(pom)
+ task.moduleMetadata.from(module)
+ task.validate()
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @Test
+ fun acceptsClosedGraphAndThirdPartyDependencies() {
+ verify(
+ "<dependency><groupId>example.pulsar</groupId><artifactId>client</artifactId>" +
+ "<version>1</version></dependency>",
+ """{"group":"third.party","module":"library","version":{"requires":"2"}}"""
+ )
+ }
+
+ @Test
+ fun rejectsMissingPomDependency() {
+ val error = expectThrows(GradleException::class.java) {
+ verify("<dependency><groupId>example.pulsar</groupId><artifactId>forgotten</artifactId>" +
+ "<version>1</version></dependency>")
+ }
+ assertTrue(error.message!!.contains("client (pom.xml) -> example.pulsar:forgotten:1"))
+ }
+
+ @Test
+ fun rejectsMissingMetadataDependency() {
+ val error = expectThrows(GradleException::class.java) {
+ verify(metadataDependency =
+ """{"group":"example.pulsar","module":"forgotten","version":{"requires":"1"}}""")
+ }
+ assertTrue(error.message!!.contains("client (Gradle metadata) -> example.pulsar:forgotten:1"))
+ }
+
+ @Test
+ fun rejectsOriginalGroupAndMissingParent() {
+ val error = expectThrows(GradleException::class.java) {
+ verify(parent = "<parent><groupId>org.apache.pulsar</groupId><artifactId>pulsar</artifactId>" +
+ "<version>1</version></parent>")
+ }
+ assertTrue(error.message!!.contains("org.apache.pulsar:pulsar:1"))
+ }
+
+ @Test
+ fun rejectsWrongInternalVersion() {
+ val error = expectThrows(GradleException::class.java) {
+ verify(metadataDependency =
+ """{"group":"example.pulsar","module":"client","version":{"strictly":"2"}}""")
+ }
+ assertTrue(error.message!!.contains("example.pulsar:client:2"))
+ }
+}
diff --git a/buildtools/build.gradle.kts b/buildtools/build.gradle.kts
index 8496a0bef970..af4dc417f66f 100644
--- a/buildtools/build.gradle.kts
+++ b/buildtools/build.gradle.kts
@@ -21,6 +21,11 @@ plugins {
id("pulsar.java-conventions")
}
+// External consumers may use the test support classes without publishing them in normal releases.
+if (PulsarApiSpiPublication.isEnabled(project)) {
+ apply(plugin = "pulsar.publish-conventions")
+}
+
dependencies {
implementation(libs.slog)
implementation(libs.snakeyaml)
diff --git a/pulsar-bom/build.gradle.kts b/pulsar-bom/build.gradle.kts
index d8ec476f4ffc..07c69fd698ab 100644
--- a/pulsar-bom/build.gradle.kts
+++ b/pulsar-bom/build.gradle.kts
@@ -33,84 +33,91 @@ javaPlatform {
dependencies {
constraints {
+ fun selectedApi(dependency: ProjectDependency) {
+ if (!PulsarApiSpiPublication.isEnabled(project) ||
+ dependency.path in PulsarApiSpiPublication.projects) {
+ add("api", dependency)
+ }
+ }
+
// Client API
- api(project(":pulsar-client-api"))
- api(project(":pulsar-client-admin-api"))
- api(project(":pulsar-client-api-v5"))
+ selectedApi(project(":pulsar-client-api"))
+ selectedApi(project(":pulsar-client-admin-api"))
+ selectedApi(project(":pulsar-client-api-v5"))
// Focused SPI modules (PIP-478): TLS factory SPI + HTTP client SPI
- api(project(":pulsar-tls-factory-api"))
- api(project(":pulsar-http-client-api"))
+ selectedApi(project(":pulsar-tls-factory-api"))
+ selectedApi(project(":pulsar-http-client-api"))
// Shaded clients (the published artifacts users depend on)
- api(project(":pulsar-client-shaded"))
- api(project(":pulsar-client-admin-shaded"))
- api(project(":pulsar-client-all"))
- api(project(":pulsar-client-v5-shaded"))
+ selectedApi(project(":pulsar-client-shaded"))
+ selectedApi(project(":pulsar-client-admin-shaded"))
+ selectedApi(project(":pulsar-client-all"))
+ selectedApi(project(":pulsar-client-v5-shaded"))
// Combined unshaded v4/v5 client and admin
- api(project(":pulsar-client-v5-all"))
+ selectedApi(project(":pulsar-client-v5-all"))
// Original (unshaded) clients
- api(project(":pulsar-client-v5"))
- api(project(":pulsar-client-original"))
- api(project(":pulsar-client-admin-original"))
+ selectedApi(project(":pulsar-client-v5"))
+ selectedApi(project(":pulsar-client-original"))
+ selectedApi(project(":pulsar-client-admin-original"))
// Client auth
- api(project(":pulsar-client-auth-sasl"))
- api(project(":pulsar-client-messagecrypto-bc"))
+ selectedApi(project(":pulsar-client-auth-sasl"))
+ selectedApi(project(":pulsar-client-messagecrypto-bc"))
// Common
- api(project(":pulsar-common"))
- api(project(":pulsar-config-validation"))
+ selectedApi(project(":pulsar-common"))
+ selectedApi(project(":pulsar-config-validation"))
// Functions API
- api(project(":pulsar-functions:pulsar-functions-api"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-api"))
// IO core
- api(project(":pulsar-io:pulsar-io-core"))
- api(project(":pulsar-io:pulsar-io-common"))
+ selectedApi(project(":pulsar-io:pulsar-io-core"))
+ selectedApi(project(":pulsar-io:pulsar-io-common"))
// Broker
- api(project(":pulsar-broker"))
- api(project(":pulsar-broker-common"))
- api(project(":pulsar-broker-auth-oidc"))
- api(project(":pulsar-broker-auth-sasl"))
+ selectedApi(project(":pulsar-broker"))
+ selectedApi(project(":pulsar-broker-common"))
+ selectedApi(project(":pulsar-broker-auth-oidc"))
+ selectedApi(project(":pulsar-broker-auth-sasl"))
// Other core modules
- api(project(":managed-ledger"))
- api(project(":pulsar-metadata"))
- api(project(":pulsar-proxy"))
- api(project(":pulsar-websocket"))
- api(project(":pulsar-testclient"))
- api(project(":pulsar-cli-utils"))
- api(project(":pulsar-client-tools"))
- api(project(":pulsar-client-tools-api"))
- api(project(":pulsar-opentelemetry"))
- api(project(":testmocks"))
+ selectedApi(project(":managed-ledger"))
+ selectedApi(project(":pulsar-metadata"))
+ selectedApi(project(":pulsar-proxy"))
+ selectedApi(project(":pulsar-websocket"))
+ selectedApi(project(":pulsar-testclient"))
+ selectedApi(project(":pulsar-cli-utils"))
+ selectedApi(project(":pulsar-client-tools"))
+ selectedApi(project(":pulsar-client-tools-api"))
+ selectedApi(project(":pulsar-opentelemetry"))
+ selectedApi(project(":testmocks"))
// Transaction
- api(project(":pulsar-transaction:pulsar-transaction-common"))
- api(project(":pulsar-transaction:pulsar-transaction-coordinator"))
+ selectedApi(project(":pulsar-transaction:pulsar-transaction-common"))
+ selectedApi(project(":pulsar-transaction:pulsar-transaction-coordinator"))
// Functions
- api(project(":pulsar-functions:pulsar-functions-instance"))
- api(project(":pulsar-functions:pulsar-functions-runtime"))
- api(project(":pulsar-functions:pulsar-functions-worker"))
- api(project(":pulsar-functions:pulsar-functions-local-runner-original"))
- api(project(":pulsar-functions:pulsar-functions-proto"))
- api(project(":pulsar-functions:pulsar-functions-secrets"))
- api(project(":pulsar-functions:pulsar-functions-utils"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-instance"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-runtime"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-worker"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-local-runner-original"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-proto"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-secrets"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-utils"))
// Athenz auth
- api(project(":pulsar-client-auth-athenz"))
- api(project(":pulsar-broker-auth-athenz"))
+ selectedApi(project(":pulsar-client-auth-athenz"))
+ selectedApi(project(":pulsar-broker-auth-athenz"))
// Functions
- api(project(":pulsar-functions:pulsar-functions-local-runner-shaded"))
+ selectedApi(project(":pulsar-functions:pulsar-functions-local-runner-shaded"))
// Tiered storage
- api(project(":tiered-storage:tiered-storage-jcloud"))
- api(project(":tiered-storage:tiered-storage-file-system"))
+ selectedApi(project(":tiered-storage:tiered-storage-jcloud"))
+ selectedApi(project(":tiered-storage:tiered-storage-file-system"))
}
}
diff --git a/pulsar-client/build.gradle.kts b/pulsar-client/build.gradle.kts
index 6c802c39735b..d5c4982b5176 100644
--- a/pulsar-client/build.gradle.kts
+++ b/pulsar-client/build.gradle.kts
@@ -164,7 +164,7 @@ tasks.named("compileTestJava") { dependsOn(generateTestAvro) }
run {
val fromGroup = "it.unimi.dsi"
val fromName = "fastutil"
- val toGroup = "org.apache.pulsar"
+ val toGroup = project.group.toString()
val toName = "pulsar-client-fastutil-minimized"
val toVersion = version.toString()
diff --git a/pulsar-functions/localrun-shaded/build.gradle.kts b/pulsar-functions/localrun-shaded/build.gradle.kts
index 598ed425bc81..52a759f5ecbe 100644
--- a/pulsar-functions/localrun-shaded/build.gradle.kts
+++ b/pulsar-functions/localrun-shaded/build.gradle.kts
@@ -27,12 +27,13 @@ dependencies {
}
val shadePrefix = "org.apache.pulsar.functions.runtime.shaded"
+val pulsarGroupPattern = Regex.escape(project.group.toString())
tasks.shadowJar {
isZip64 = true
dependencies {
- include(dependency("org.apache.pulsar:.*"))
+ include(dependency("$pulsarGroupPattern:.*"))
include(project(":pulsar-functions:pulsar-functions-local-runner-original"))
include(project(":pulsar-client-original"))
include(project(":pulsar-common"))