You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2026/09/16 00:34:05 UTC
(skywalking-nodejs) branch master updated: feat: add gRPC mutual TLS support for the OAP channel (#146)
This is an automated email from the ASF dual-hosted git repository.
wu-sheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking-nodejs.git
The following commit(s) were added to refs/heads/master by this push:
new d7f266f feat: add gRPC mutual TLS support for the OAP channel (#146)
d7f266f is described below
commit d7f266f9bad4f55a5d260377abe7a4a26ce7eaaf
Author: songzhendong <xi...@qq.com>
AuthorDate: Wed Sep 16 08:34:00 2026 +0800
feat: add gRPC mutual TLS support for the OAP channel (#146)
---
.gitattributes | 11 ++
.github/workflows/test.yaml | 35 ++++
.gitignore | 6 +
.licenserc.yaml | 3 +
docs/en/advanced/troubleshooting.md | 20 ++-
docs/en/contribution/build-and-test.md | 13 ++
docs/en/setup/configuration.md | 38 ++++-
src/agent/core/remote/TLSChannelBuilder.ts | 71 +++++++-
src/config/AgentConfig.ts | 8 +
tests/remote-e2e/mtls/README.md | 29 ++++
tests/remote-e2e/mtls/client/ca.crt | 29 ++++
tests/remote-e2e/mtls/client/client.crt | 30 ++++
tests/remote-e2e/mtls/docker-compose.yml | 99 ++++++++++++
tests/remote-e2e/mtls/generate-certs.sh | 77 +++++++++
tests/remote-e2e/mtls/server.ts | 71 ++++++++
tests/remote-e2e/mtls/server/ca.crt | 29 ++++
tests/remote-e2e/mtls/server/server.crt | 30 ++++
tests/remote-e2e/mtls/test.ts | 252 +++++++++++++++++++++++++++++
tests/remote/GRPCChannelManager.test.ts | 8 +
tests/remote/TLSChannelBuilder.test.ts | 174 ++++++++++++++++++++
20 files changed, 1021 insertions(+), 12 deletions(-)
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..d8e2c1f
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,11 @@
+# Keep LF in paths this PR touches. CRLF in *.sh breaks Linux CI (`$'\r': command not found`).
+*.sh text eol=lf
+.github/workflows/*.yml text eol=lf
+.github/workflows/*.yaml text eol=lf
+.gitignore text eol=lf
+.licenserc.yaml text eol=lf
+.gitattributes text eol=lf
+docs/**/*.md text eol=lf
+tests/remote-e2e/mtls/** text eol=lf
+*.crt -text
+*.pem -text
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 78930dd..5dadc6a 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -150,6 +150,41 @@ jobs:
npm i
npm run test tests/remote-e2e/static-failover/
+ TestRemoteE2EMTLS:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ node-version: [ 20, 22, 24 ]
+ env:
+ SW_NODE_VERSION: ${{ matrix.node-version }}
+ SW_OAP_IMAGE: ghcr.io/apache/skywalking/oap@sha256:f3fd1982c0344db2150780ea71bd51afc1f8fd7f482f73f01429a72151c857f6
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: true
+
+ - name: Generate mTLS test certificates
+ run: bash tests/remote-e2e/mtls/generate-certs.sh
+
+ - uses: actions/cache@v4
+ with:
+ path: ~/.npm
+ key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+ restore-keys: |
+ ${{ runner.os }}-node-
+
+ - name: Set Up NodeJS ${{ matrix.node-version }}
+ uses: actions/setup-node@v4
+ with:
+ node-version: ${{ matrix.node-version }}
+
+ - name: Remote E2E On Node@${{ matrix.node-version }} (real OAP mTLS)
+ run: |
+ npm i
+ npm run test tests/remote-e2e/mtls/
+
TestLib:
runs-on: ubuntu-latest
timeout-minutes: 30
diff --git a/.gitignore b/.gitignore
index e9bf187..f09a33f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,9 @@ scripts/.finalize-work/
skywalking-nodejs-src-*.tgz
skywalking-nodejs-src-*.tgz.asc
skywalking-nodejs-src-*.tgz.sha512
+
+# mTLS e2e test private keys (generated by generate-certs.sh, never commit)
+tests/remote-e2e/mtls/**/*.pem
+tests/remote-e2e/mtls/**/*.key
+tests/remote-e2e/mtls/**/*.srl
+tests/remote-e2e/mtls/**/*.csr
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 81961c3..cb142b9 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -29,11 +29,14 @@ header:
- 'LICENSE'
- 'NOTICE'
- '.gitignore'
+ - '.gitattributes'
- '.gitmodules'
- '.prettierrc'
- 'lib'
- '.husky'
- '.file-headerrc'
- '.eslintrc.js'
+ - '**/*.crt'
+ - '**/*.pem'
comment: on-failure
diff --git a/docs/en/advanced/troubleshooting.md b/docs/en/advanced/troubleshooting.md
index 5636867..a662a4f 100644
--- a/docs/en/advanced/troubleshooting.md
+++ b/docs/en/advanced/troubleshooting.md
@@ -40,9 +40,18 @@ An option passed to `agent.start()` replaces the environment value. This include
agent.start({ collectorAddress: '' });
```
-For a TLS OAP endpoint, set `SW_AGENT_SECURE=true`. The agent uses the system trust store. It has no
-configuration for a custom CA or client certificate. Set `SW_AGENT_AUTHENTICATION` if OAP requires
-an agent token.
+For a TLS OAP endpoint, set `SW_AGENT_SECURE=true`. The agent uses the system trust store by
+default. Set `SW_AGENT_AUTHENTICATION` if OAP requires an agent token.
+
+For a custom CA or mTLS, configure `SW_AGENT_SSL_TRUSTED_CA_PATH`. For mTLS, also configure both
+`SW_AGENT_SSL_KEY_PATH` and `SW_AGENT_SSL_CERT_CHAIN_PATH`; all three paths are optional relative
+to the Node.js process working directory. The client key and certificate must be a matching pair.
+The agent fails channel creation when either client path is missing or unreadable, rather than
+silently falling back to one-way TLS.
+
+With Apache OAP, point the agent at the mTLS-enabled `receiver-sharing-server` gRPC listener
+(commonly port `11801`, or the port selected by `SW_RECEIVER_GRPC_PORT`). Do not assume that the
+regular OAP agent listener on port `11800` requests client certificates.
Under TLS with multiple hostnames, certificate verification follows the channel authority (the first
list entry in the configured target). Endpoint pick order may be shuffled by grpc-js, but the target
@@ -50,6 +59,11 @@ string — and therefore authority / SNI — stays in config order. Every backen
certificate that shares the needed SANs, or failover handshakes fail. Prefer one DNS name with
multiple A/AAAA records for TLS high availability.
+If the agent connects with a hostname or IP that does not appear in the OAP certificate SAN, the
+handshake fails with a hostname mismatch. Set `SW_AGENT_SSL_TARGET_NAME_OVERRIDE` to the SAN name
+that OAP does present — for example the DNS name when the agent connects via an IP literal.
+This overrides both the TLS SNI and the gRPC default authority.
+
Channel disconnect lines are logged at `error` and recover lines at `warn` (throttled separately so a
recover line is not swallowed by the disconnect window). For per-address grpc-js detail, set
`GRPC_TRACE=pick_first,subchannel`.
diff --git a/docs/en/contribution/build-and-test.md b/docs/en/contribution/build-and-test.md
index 586335c..80096ad 100644
--- a/docs/en/contribution/build-and-test.md
+++ b/docs/en/contribution/build-and-test.md
@@ -62,6 +62,18 @@ npm run test tests/plugins/http/
Plugin tests start a mock SkyWalking collector and target services with Docker Compose. Make sure
Docker is running and that the test ports are free.
+### Real OAP mTLS remote e2e
+
+The suite under `tests/remote-e2e/mtls/` talks to a real OAP over mutual TLS. Private keys are not
+in git — generate them first (needs `openssl` and `bash`):
+
+```bash
+bash tests/remote-e2e/mtls/generate-certs.sh
+npm run test tests/remote-e2e/mtls/
+```
+
+See `tests/remote-e2e/mtls/README.md` for details. CI runs the same script before the Jest job.
+
Test the built package in a container:
```bash
@@ -81,6 +93,7 @@ docker run --rm skywalking-nodejs:test
| `src/trace/` | Trace context, spans, segments, and component IDs |
| `src/aws/`, `src/azure/` | Serverless wrappers and AWS helpers |
| `tests/plugins/` | Docker-based plugin integration tests |
+| `tests/remote-e2e/` | Real-collector / real-OAP remote e2e (incl. mTLS) |
| `tests/remote/`, `tests/runtime/` | Reporter and runtime meter tests |
## Before opening a pull request
diff --git a/docs/en/setup/configuration.md b/docs/en/setup/configuration.md
index e4d77b3..15f8d35 100644
--- a/docs/en/setup/configuration.md
+++ b/docs/en/setup/configuration.md
@@ -26,14 +26,48 @@ Node.js process. `SW_DISABLE` is checked when `start()` is called.
| `SW_AGENT_INSTANCE` | `serviceInstance` | Host name | Service instance name shown in SkyWalking. |
| `SW_AGENT_COLLECTOR_BACKEND_SERVICES` | `collectorAddress` | `127.0.0.1:11800` | OAP gRPC address(es). One `host:port` uses grpc-js `dns:` (all A/AAAA become endpoints; periodically re-resolved). A comma-separated list uses a static resolver with `pick_first` (literal endpoints only; no per-name DNS expansion or re-resolution). Endpoint pick order is shuffled by grpc-js (`shuffleAddressList`); the target string keeps config order so TLS authority / SNI stay on the first list entry. Und [...]
| `SW_AGENT_SECURE` | `secure` | `false` | Use TLS for the OAP gRPC connection. |
+| `SW_AGENT_SSL_TRUSTED_CA_PATH` | `sslTrustedCaPath` | Not set | Optional trusted CA certificate file for TLS server verification. Relative paths are resolved from the process working directory. [...]
+| `SW_AGENT_SSL_KEY_PATH` | `sslKeyPath` | Not set | Client private key file for mTLS. Must be configured together with `sslCertChainPath` and `secure=true`. [...]
+| `SW_AGENT_SSL_CERT_CHAIN_PATH` | `sslCertChainPath` | Not set | Client certificate chain file for mTLS. Must be configured together with `sslKeyPath` and `secure=true`. [...]
+| `SW_AGENT_SSL_TARGET_NAME_OVERRIDE` | `sslTargetNameOverride` | Not set | Override the hostname used for TLS server name indication (SNI) and certificate verification. Useful when the OAP certificate SAN does not include the literal hostname or IP the agent connects to. [...]
| `SW_AGENT_AUTHENTICATION` | `authorization` | Not set | Authentication token sent to OAP. |
| `SW_AGENT_TRACE_TIMEOUT` | `traceTimeout` | `10000` | gRPC deadline in milliseconds for trace and meter reports and service management requests. Must be a positive integer. |
For token authentication, set the same token in OAP with `SW_AUTHENTICATION`. See
[OAP token authentication](https://skywalking.apache.org/docs/main/next/en/setup/backend/backend-token-auth/).
-When `secure` is enabled, the agent uses the system trust store. It does not provide options for a
-custom CA, client certificate, or mutual TLS.
+When `secure` is enabled, the agent uses the system trust store unless `sslTrustedCaPath` is set.
+Configure both `sslKeyPath` and `sslCertChainPath` to enable mTLS. The private key and certificate
+files are read when the gRPC channel is created. Incomplete or unreadable certificate configuration
+fails closed instead of silently downgrading to one-way TLS.
+
+If the OAP server certificate SAN does not contain the hostname or IP that the agent connects to —
+for example when the agent reaches OAP via an IP literal but the certificate is issued for a DNS
+name — set `sslTargetNameOverride` to the SAN value that the certificate does include. This is
+equivalent to grpc-js `grpc.ssl_target_name_override` and also updates `grpc.default_authority`.
+
+Relative certificate paths are resolved from the process working directory. Prefer absolute paths
+(`/etc/sw/ca.crt`) or ensure the working directory is stable (e.g. Docker containers with WORKDIR
+set). The agent accepts both PKCS#1 (`-----BEGIN RSA PRIVATE KEY-----`) and PKCS#8
+(`-----BEGIN PRIVATE KEY-----`) private key formats via Node's OpenSSL-backed `crypto` module.
+
+For Apache OAP mTLS, enable the `receiver-sharing-server` gRPC listener with its trusted client CA
+and point `collectorAddress` to that listener (commonly port `11801`, or the port selected by
+`SW_RECEIVER_GRPC_PORT`). The standard OAP mTLS setup does not use the regular agent listener on
+port `11800`.
+
+For example:
+
+```typescript
+agent.start({
+ collectorAddress: 'oap.example.com:11801',
+ secure: true,
+ sslTrustedCaPath: './certs/ca.crt',
+ sslKeyPath: './certs/client.key',
+ sslCertChainPath: './certs/client.crt',
+});
+```
+
## Agent control and logging
diff --git a/src/agent/core/remote/TLSChannelBuilder.ts b/src/agent/core/remote/TLSChannelBuilder.ts
index d3d7e51..056b465 100644
--- a/src/agent/core/remote/TLSChannelBuilder.ts
+++ b/src/agent/core/remote/TLSChannelBuilder.ts
@@ -17,19 +17,76 @@
*
*/
+import * as fs from 'fs';
+import * as path from 'path';
import * as grpc from '@grpc/grpc-js';
+import { ChannelOptions } from '@grpc/grpc-js';
import config from '../../../config/AgentConfig';
+import { createLogger } from '../../../logging';
import ChannelBuilder, { ChannelBuildContext } from './ChannelBuilder';
-/** When SW_AGENT_SECURE=true, upgrade channel credentials to TLS (Java TLSChannelBuilder simplified). */
+const logger = createLogger(__filename);
+
+function configuredPath(value: string | undefined): string | undefined {
+ const trimmed = value?.trim();
+ return trimmed ? path.resolve(trimmed) : undefined;
+}
+
+function readCertificateFile(filePath: string, description: string): Buffer {
+ try {
+ return fs.readFileSync(filePath);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ throw new Error(`Failed to read ${description} at [${filePath}]: ${message}`);
+ }
+}
+
+/**
+ * Build TLS or mTLS credentials from the agent configuration.
+ * Client authentication is enabled only when both the private key and cert chain are configured.
+ */
export default class TLSChannelBuilder implements ChannelBuilder {
build(context: ChannelBuildContext): ChannelBuildContext {
- if (config.secure) {
- return {
- ...context,
- credentials: grpc.credentials.createSsl(),
- };
+ const trustedCaPath = configuredPath(config.sslTrustedCaPath);
+ const keyPath = configuredPath(config.sslKeyPath);
+ const certChainPath = configuredPath(config.sslCertChainPath);
+ const hasTlsMaterial = trustedCaPath || keyPath || certChainPath;
+
+ if (!config.secure) {
+ if (hasTlsMaterial) {
+ throw new Error('TLS certificate configuration requires secure=true');
+ }
+ return context;
}
- return context;
+
+ if (Boolean(keyPath) !== Boolean(certChainPath)) {
+ throw new Error('Both sslKeyPath and sslCertChainPath must be configured to enable mTLS');
+ }
+
+ const rootCerts = trustedCaPath ? readCertificateFile(trustedCaPath, 'the trusted CA certificate') : undefined;
+ const privateKey = keyPath ? readCertificateFile(keyPath, 'the client private key') : undefined;
+ const certChain = certChainPath ? readCertificateFile(certChainPath, 'the client certificate chain') : undefined;
+
+ const credentials = grpc.credentials.createSsl(rootCerts, privateKey, certChain);
+
+ logger.debug(
+ `gRPC TLS credentials built: ca=${trustedCaPath ?? '(system)'} key=${keyPath ?? '(none)'} cert=${
+ certChainPath ?? '(none)'
+ }`,
+ );
+
+ const sslTargetNameOverride = config.sslTargetNameOverride?.trim();
+ const extraOptions: ChannelOptions = {};
+ if (sslTargetNameOverride) {
+ extraOptions['grpc.ssl_target_name_override'] = sslTargetNameOverride;
+ extraOptions['grpc.default_authority'] = sslTargetNameOverride;
+ logger.debug(`gRPC TLS hostname override set to [${sslTargetNameOverride}]`);
+ }
+
+ return {
+ ...context,
+ credentials,
+ options: { ...context.options, ...extraOptions },
+ };
}
}
diff --git a/src/config/AgentConfig.ts b/src/config/AgentConfig.ts
index 9bea217..b46c860 100644
--- a/src/config/AgentConfig.ts
+++ b/src/config/AgentConfig.ts
@@ -24,6 +24,10 @@ export type AgentConfig = {
serviceInstance?: string;
collectorAddress?: string;
secure?: boolean;
+ sslTrustedCaPath?: string;
+ sslKeyPath?: string;
+ sslCertChainPath?: string;
+ sslTargetNameOverride?: string;
authorization?: string;
maxBufferSize?: number;
coldEndpoint?: boolean;
@@ -182,6 +186,10 @@ const _config = {
})(),
collectorAddress: process.env.SW_AGENT_COLLECTOR_BACKEND_SERVICES || '127.0.0.1:11800',
secure: process.env.SW_AGENT_SECURE?.toLowerCase() === 'true',
+ sslTrustedCaPath: process.env.SW_AGENT_SSL_TRUSTED_CA_PATH || undefined,
+ sslKeyPath: process.env.SW_AGENT_SSL_KEY_PATH || undefined,
+ sslCertChainPath: process.env.SW_AGENT_SSL_CERT_CHAIN_PATH || undefined,
+ sslTargetNameOverride: process.env.SW_AGENT_SSL_TARGET_NAME_OVERRIDE || undefined,
authorization: process.env.SW_AGENT_AUTHENTICATION,
maxBufferSize: ((n) => (Number.isSafeInteger(n) && n > 0 ? n : 1000))(
Number.parseInt(process.env.SW_AGENT_MAX_BUFFER_SIZE ?? '', 10),
diff --git a/tests/remote-e2e/mtls/README.md b/tests/remote-e2e/mtls/README.md
new file mode 100644
index 0000000..c1c5b2e
--- /dev/null
+++ b/tests/remote-e2e/mtls/README.md
@@ -0,0 +1,29 @@
+# Real OAP mTLS remote e2e
+
+End-to-end coverage of Agent → OAP gRPC over mutual TLS against a real OAP +
+BanyanDB stack (see `docker-compose.yml`).
+
+## Prerequisites
+
+- Docker
+- Node.js (same majors as CI: 20 / 22 / 24)
+- `openssl` and `bash` (Git Bash or WSL on Windows)
+
+## Generate certificates
+
+Private keys are not committed. Generate them before running the suite:
+
+```bash
+bash tests/remote-e2e/mtls/generate-certs.sh
+```
+
+This writes `server/server.pem`, `client/client.pem`, and refreshes the CA /
+leaf certificates under `server/` and `client/`. Those `.pem` files are gitignored.
+
+## Run
+
+```bash
+npm run test tests/remote-e2e/mtls/
+```
+
+CI runs the same `generate-certs.sh` step before the Jest suite.
diff --git a/tests/remote-e2e/mtls/client/ca.crt b/tests/remote-e2e/mtls/client/ca.crt
new file mode 100644
index 0000000..4877512
--- /dev/null
+++ b/tests/remote-e2e/mtls/client/ca.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6DCCAtCgAwIBAgIUckh59cIFCT3jBextzlcwZ/bkrKswDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPc3ctbXRscy10ZXN0LWNhMB4XDTI2MDkxNTA2NDczM1oX
+DTM2MDkxMjA2NTIzM1owGjEYMBYGA1UEAwwPc3ctbXRscy10ZXN0LWNhMIICIjAN
+BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAwLZ4PzlE2nSSZO+SePU1Rll6gkcQ
+QvDAOZR8o+3IQPGL8g4oZInJreTvwavVxpT1n+MlCZRPLrngpGg7+c33+FNhDmIm
+sl19WgknMk82ISTKVEaMmEXq/bKko3VBsi8GIAq2rsTfaEV4wt0SoX+sMagg6FWw
+wRiYEJgn01F7t1R9nc+/18VGpPGpk7CK+u6UfHCAHUFU0IUmPXHlM4ZO1GrO0F/3
+E9oIyr1xWAJtbbQ9lDb2W9z+KRDJPUIZzuE2ffIYIv+hGWxU4guzoCYFCGoYW/Rp
+C1A3rXTza6HDs0nBA2pnVNV0j25Z6JA8PEqRnunxc9Q6TYuz7SxBmUelnHN77iu5
+ZT8Hg5fto1+kuAXXFr4YqpZlSTzDtlRW1Ts4hROk01ZpRpt4P9fhxUGJU9XwHWcj
+fWv8U5yJZFqyWf/ubf12ybqbvBYutj5iVtxkz3KayHN88THwigZg7Epy+tGjE+EU
+rvrC1o7My1hOb6ZwEUQBpeQ3b/w9Fq1AK4aAkrYp8dx/DYDYuefNseYKkb41UV2r
+93WLUqdVxbPiZEeCEcnDgzmSI6BFyd78s3WsjD5CYj4TvV3XrfeIXXvWsPSqovY3
+mHu1/Ye6y8vBal5TkN/+tYVOL/D9TQD/W/VjLD/HbppuKJk/IRNf93RgxhVEt+6g
+NuPVyedFqD3ADgMCAwEAAaMmMCQwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8B
+Af8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAHQfCal9ZlCKu+/J9GKpJvovmAlu
+LlRuGurXPH7pN3IPZo+AAxyzTbDIX9TU4IPOgfhL+k4OKBs0R5PmST7DBn/Q4Ile
+4c/EPuY3iDS+RrTebgDQ+zPrkeJ30SAK5wG/ZDNPqaBNxHaIepZnh0qb+65H10e7
+PuecuL8U0ZrpfhrXPodYgYPTM5tf8A7tEJKnx8AjKb4a0zoSq/0Wk9ScnNxi02Is
+yvqwhTK7RZX/TlJlrIXTiDTFDT9RS4dmvSaWurrDM7qcUkFkqErnBX+3BREJ5m6g
+pmGUdoRq+D9KWhnas5ATJB2sExzP/7J2JTA01/5KHBbg+k064DE/N70Cvh+10x0V
+pSXJd4BgE7aRjB3BClGSvGLve+KKRSxU9I/6GHMmW2N9/t1IJau4QjqlnxggDqlu
+r6VwCBVPuOUbR0K0pXdiILcFrGYg3OhYnjqti/IHPvSmmILzNIDknEsuTlw4lwlE
+Oqhul9wh6EJH/JTCydMg/0twrn8CHTcNzX6VDpXb16F+d65HnOo5jeabAOjWVlCb
+b42qOUsUqSckQs8Ag7aLnhhq7s99q19hZoC9BjS9dFEB2iPzq4wa/kO4gsVrLSBv
+pAQMz3/DnLV/G81l2RnMotJi2s9S4t18uQTjsyPID5Uh0uT1rwGxDGhkCp7PhrsD
+RXjsLdjVuANlkHYs
+-----END CERTIFICATE-----
diff --git a/tests/remote-e2e/mtls/client/client.crt b/tests/remote-e2e/mtls/client/client.crt
new file mode 100644
index 0000000..6a610ff
--- /dev/null
+++ b/tests/remote-e2e/mtls/client/client.crt
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFGzCCAwOgAwIBAgIUZ8coAXTH7vRCPc1VeMw3MPvtGiQwDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPc3ctbXRscy10ZXN0LWNhMB4XDTI2MDkxNTA2NDczNVoX
+DTM2MDkxMjA2NTIzNVowGDEWMBQGA1UEAwwNc3ctbm9kZS1hZ2VudDCCAiIwDQYJ
+KoZIhvcNAQEBBQADggIPADCCAgoCggIBAKbtLmGtHDBAO3oUvHn9P9zmR2W8tx24
+1OKOXkJNwpYc2ud6IyspbQCml/vsX5jq183mhrC5ARt9kjt5J8ZrBOgtn3wgjzVv
+pFQfjPKaKv5iR2AZzdvdBztzUZeUZmqVKWiAkVrapVyd4Z14LCLwXv89rz4AgSMR
+BCm4V/8hhkO4ynSay46tql7/NB8O406Om7rTA07XwZh8Q8xav1eBW7/tMWXSuO1j
+/Uu2nQ6hLcQTu/ENgpQsI4j2Vifm9taqgXRpP2KCwZODqBZiHSCRbPIfAklIeVM+
+0fGAoY3UV/XiObD13PqZu5rneybv9IuK37haNFkzXbR9w7K0SWZNBzFymva18+eN
+FXsZhsfQMTcX0hZkFVQC53Qw0nOfhhMnVmFeFmRP7QC+7pS39VNBD8jB05K8wrUg
+qZ/ADexxqzJPc1e6EWmZqcvqJC2w4cIBiXhuQAwts54NR1twAzVnRE4j05C99ajg
+rmFRRPLOHZ9XogQH+nbJJ2ZyWRXFZb4JC71S2nULXiIIRwa0aRUyrL/yG4n5p6Rd
+QuNCBMXrgf/zjTkBMJcgFSNvsGVoHE6txSv6L4UfMBzoEmjvfQmZzz5YwH+oQ7N5
+c1AEqQPmlAEqC8gQQpuL3sWNXCNLcZgJ5Qu0OUrciuAiAE1bGRWPrsgHhFdMHvMl
+u/3xlNRGypW/AgMBAAGjWzBZMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgWg
+MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAaBgNVHREEEzARgglsb2Nh
+bGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggIBACsv8Gbbx81BfrWsi+lCzDYw
+F4D1YtFCc9IZStMj3m/aAxH/HjY8Sm04YCn6q56B3B+kY7jPfGCYChqNYwqNjhKO
+C/nKBFkhvNtP57+5btmOetWI0y36yTeuoc0ATHeW6aZihmDn6fsbVKwg6qJYB2Zm
+2InLsdFHZffj/sLqqhKeThNF7tyt92uSh6y8/mAbTyDaz/YItr/V9rUHRFvLQ46r
+jSJxe3VbUuTFmh5/o5eURNewYTGz1YnFyNs33J8riG8eWaxyYCh3yw1vk4tj1nxC
+kJhylAym8Qbka71sBU32ptq26wFKsohbzHoiY9oPr092CJ2Yql5O6j7qtTAo9s+4
+qvqqpcMQOC0jtA4DuYipy8JgW5/Hbx4WbQ/ybJL8EzEdJMwOrhjm3sFihXkT0o4q
+B76mRs3KusnvXsJIyyYgmF6vlzgaTyKTWPXgr49VxCgsKEoZCdyTTCeOODfEorcR
+MZb7TDL+NMHhxAJzQzD2MGrR7LfFAtN9ziq5bRub+WIbtUjEPr84Djr0Ah2HDYaA
+uQkX8xWCx2xICfUbsXiFl3jUXVPqTvDFRXD2Tieiz4jYeg1w5TMapsevlFI7/9ci
+y32gvZW4oTOvwSZ/v/mhZSWA/iGuJXEUVeaNM4eC5zwPCnsFJcPiAeJV5H1K9pG4
+L7k3VHYUnfo1SHh9kx7V
+-----END CERTIFICATE-----
diff --git a/tests/remote-e2e/mtls/docker-compose.yml b/tests/remote-e2e/mtls/docker-compose.yml
new file mode 100644
index 0000000..941178d
--- /dev/null
+++ b/tests/remote-e2e/mtls/docker-compose.yml
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+services:
+ oap:
+ image: ${SW_OAP_IMAGE:-ghcr.io/apache/skywalking/oap@sha256:f3fd1982c0344db2150780ea71bd51afc1f8fd7f482f73f01429a72151c857f6}
+ volumes:
+ - ./server:/skywalking/certs:ro
+ environment:
+ SW_STORAGE: banyandb
+ SW_STORAGE_BANYANDB_TARGETS: banyandb:17912
+ SW_RECEIVER_GRPC_PORT: 11811
+ SW_RECEIVER_GRPC_SSL_ENABLED: 'true'
+ SW_RECEIVER_GRPC_SSL_KEY_PATH: /skywalking/certs/server.pem
+ SW_RECEIVER_GRPC_SSL_CERT_CHAIN_PATH: /skywalking/certs/server.crt
+ SW_RECEIVER_GRPC_SSL_TRUSTED_CAS_PATH: /skywalking/certs/ca.crt
+ ports:
+ - '11811:11811'
+ - '12801:12800'
+ expose:
+ - '11811'
+ depends_on:
+ banyandb:
+ condition: service_healthy
+ healthcheck:
+ test: ['CMD', 'bash', '-c', 'cat < /dev/null > /dev/tcp/127.0.0.1/11811']
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ start_period: 90s
+ networks:
+ - mtls
+
+ banyandb:
+ image: ${SW_BANYANDB_IMAGE:-ghcr.io/apache/skywalking-banyandb:3b83e18fb0481d02e44eaa5df137fcf7b000754b}
+ command:
+ - standalone
+ - --measure-metadata-cache-wait-duration
+ - 1m
+ - --stream-metadata-cache-wait-duration
+ - 1m
+ expose:
+ - '17912'
+ healthcheck:
+ test: ['CMD', 'sh', '-c', 'nc -nz 127.0.0.1 17912']
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - mtls
+
+ server:
+ build:
+ context: ../../..
+ dockerfile: tests/plugins/common/Dockerfile.agent
+ args:
+ - SW_NODE_VERSION=${SW_NODE_VERSION:-22}
+ ports:
+ - '5011:5000'
+ environment:
+ SW_AGENT_NAME: nodejs-mtls-e2e
+ SW_AGENT_INSTANCE: nodejs-mtls-e2e-instance
+ SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11811
+ SW_AGENT_SECURE: 'true'
+ SW_AGENT_SSL_TRUSTED_CA_PATH: /app/certs/ca.crt
+ SW_AGENT_SSL_KEY_PATH: /app/certs/client.pem
+ SW_AGENT_SSL_CERT_CHAIN_PATH: /app/certs/client.crt
+ SW_AGENT_RUNTIME_METRICS_REPORTER_ACTIVE: 'false'
+ volumes:
+ - ./client:/app/certs:ro
+ - .:/app/tests/remote-e2e/mtls:ro
+ entrypoint: ['bash', '-c', 'npx ts-node /app/tests/remote-e2e/mtls/server.ts']
+ depends_on:
+ oap:
+ condition: service_healthy
+ healthcheck:
+ test: ['CMD', 'bash', '-c', 'cat < /dev/null > /dev/tcp/127.0.0.1/5000']
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - mtls
+
+networks:
+ mtls:
diff --git a/tests/remote-e2e/mtls/generate-certs.sh b/tests/remote-e2e/mtls/generate-certs.sh
new file mode 100644
index 0000000..0a43758
--- /dev/null
+++ b/tests/remote-e2e/mtls/generate-certs.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Generate throwaway CA / server / client material for the mTLS remote-e2e suite.
+# Private keys are written under this directory and must never be committed
+# (see .gitignore). Requires openssl and bash.
+
+set -euo pipefail
+
+# Git Bash / MSYS rewrites leading-/ arguments as Windows paths (breaks -subj /CN=...).
+export MSYS_NO_PATHCONV=1
+export MSYS2_ARG_CONV_EXCL='*'
+
+# Stay in this directory and use relative paths — MinGW openssl can fail writing
+# absolute /c/... paths on Windows Git Bash.
+cd "$(dirname "${BASH_SOURCE[0]}")"
+mkdir -p server client
+
+cleanup() {
+ rm -f server/ca.key server/ca.srl \
+ server/server.csr server/server.srl \
+ client/client.csr client/client.srl \
+ server/server-san.ext client/client-san.ext
+}
+trap cleanup EXIT
+
+# CA key + cert
+openssl genrsa -out server/ca.key 2048
+openssl req -new -x509 -days 3650 -key server/ca.key \
+ -out server/ca.crt -subj "/CN=SkyWalking Test CA"
+
+# Server key + cert signed by CA (SAN: oap + localhost + 127.0.0.1)
+# Use a real extfile — process substitution (/dev/fd/N) breaks MinGW openssl on Windows.
+printf 'subjectAltName=DNS:oap,DNS:localhost,IP:127.0.0.1\n' > server/server-san.ext
+openssl genrsa -out server/server.pem 2048
+openssl req -new -key server/server.pem \
+ -out server/server.csr -subj "/CN=oap"
+openssl x509 -req -in server/server.csr -days 3650 \
+ -CA server/ca.crt -CAkey server/ca.key -CAcreateserial \
+ -out server/server.crt \
+ -extfile server/server-san.ext
+
+# Client key + cert signed by CA (SAN: localhost + 127.0.0.1)
+printf 'subjectAltName=DNS:localhost,IP:127.0.0.1\n' > client/client-san.ext
+openssl genrsa -out client/client.pem 2048
+openssl req -new -key client/client.pem \
+ -out client/client.csr -subj "/CN=nodejs-mtls-e2e"
+openssl x509 -req -in client/client.csr -days 3650 \
+ -CA server/ca.crt -CAkey server/ca.key -CAcreateserial \
+ -out client/client.crt \
+ -extfile client/client-san.ext
+
+# Agent container trusts the same CA that signed OAP's server cert.
+cp server/ca.crt client/ca.crt
+
+# Drop intermediates and the CA private key (only leaf keys remain for the test).
+# trap cleanup also removes these on exit.
+cleanup
+trap - EXIT
+
+echo "mTLS e2e certificates written under $(pwd)/{server,client}/"
+ls -la server/ client/
diff --git a/tests/remote-e2e/mtls/server.ts b/tests/remote-e2e/mtls/server.ts
new file mode 100644
index 0000000..08b3ec7
--- /dev/null
+++ b/tests/remote-e2e/mtls/server.ts
@@ -0,0 +1,71 @@
+/*!
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import agent from '../../../src';
+import * as http from 'http';
+
+agent.start({ serviceName: 'nodejs-mtls-e2e', maxBufferSize: 1000 });
+
+const server = http.createServer((req, res) => {
+ if (req.url === '/ping') {
+ res.statusCode = 200;
+ res.end('ok');
+ return;
+ }
+
+ if (req.url === '/flush') {
+ const deadlineMs = 8000;
+ let completed = false;
+ const timer = setTimeout(() => {
+ if (completed) {
+ return;
+ }
+ completed = true;
+ res.statusCode = 504;
+ res.end('flush deadline');
+ }, deadlineMs);
+
+ void Promise.resolve(agent.flush())
+ .then(() => {
+ if (completed) {
+ return;
+ }
+ completed = true;
+ clearTimeout(timer);
+ res.statusCode = 200;
+ res.end('flushed');
+ })
+ .catch((error: unknown) => {
+ if (completed) {
+ return;
+ }
+ completed = true;
+ clearTimeout(timer);
+ const message = error instanceof Error ? error.message : String(error);
+ res.statusCode = 500;
+ res.end(`flush failed: ${message}`);
+ });
+ return;
+ }
+
+ res.statusCode = 404;
+ res.end('not found');
+});
+
+server.listen(5000, () => console.info('Listening on port 5000...'));
diff --git a/tests/remote-e2e/mtls/server/ca.crt b/tests/remote-e2e/mtls/server/ca.crt
new file mode 100644
index 0000000..4877512
--- /dev/null
+++ b/tests/remote-e2e/mtls/server/ca.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6DCCAtCgAwIBAgIUckh59cIFCT3jBextzlcwZ/bkrKswDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPc3ctbXRscy10ZXN0LWNhMB4XDTI2MDkxNTA2NDczM1oX
+DTM2MDkxMjA2NTIzM1owGjEYMBYGA1UEAwwPc3ctbXRscy10ZXN0LWNhMIICIjAN
+BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAwLZ4PzlE2nSSZO+SePU1Rll6gkcQ
+QvDAOZR8o+3IQPGL8g4oZInJreTvwavVxpT1n+MlCZRPLrngpGg7+c33+FNhDmIm
+sl19WgknMk82ISTKVEaMmEXq/bKko3VBsi8GIAq2rsTfaEV4wt0SoX+sMagg6FWw
+wRiYEJgn01F7t1R9nc+/18VGpPGpk7CK+u6UfHCAHUFU0IUmPXHlM4ZO1GrO0F/3
+E9oIyr1xWAJtbbQ9lDb2W9z+KRDJPUIZzuE2ffIYIv+hGWxU4guzoCYFCGoYW/Rp
+C1A3rXTza6HDs0nBA2pnVNV0j25Z6JA8PEqRnunxc9Q6TYuz7SxBmUelnHN77iu5
+ZT8Hg5fto1+kuAXXFr4YqpZlSTzDtlRW1Ts4hROk01ZpRpt4P9fhxUGJU9XwHWcj
+fWv8U5yJZFqyWf/ubf12ybqbvBYutj5iVtxkz3KayHN88THwigZg7Epy+tGjE+EU
+rvrC1o7My1hOb6ZwEUQBpeQ3b/w9Fq1AK4aAkrYp8dx/DYDYuefNseYKkb41UV2r
+93WLUqdVxbPiZEeCEcnDgzmSI6BFyd78s3WsjD5CYj4TvV3XrfeIXXvWsPSqovY3
+mHu1/Ye6y8vBal5TkN/+tYVOL/D9TQD/W/VjLD/HbppuKJk/IRNf93RgxhVEt+6g
+NuPVyedFqD3ADgMCAwEAAaMmMCQwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8B
+Af8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAHQfCal9ZlCKu+/J9GKpJvovmAlu
+LlRuGurXPH7pN3IPZo+AAxyzTbDIX9TU4IPOgfhL+k4OKBs0R5PmST7DBn/Q4Ile
+4c/EPuY3iDS+RrTebgDQ+zPrkeJ30SAK5wG/ZDNPqaBNxHaIepZnh0qb+65H10e7
+PuecuL8U0ZrpfhrXPodYgYPTM5tf8A7tEJKnx8AjKb4a0zoSq/0Wk9ScnNxi02Is
+yvqwhTK7RZX/TlJlrIXTiDTFDT9RS4dmvSaWurrDM7qcUkFkqErnBX+3BREJ5m6g
+pmGUdoRq+D9KWhnas5ATJB2sExzP/7J2JTA01/5KHBbg+k064DE/N70Cvh+10x0V
+pSXJd4BgE7aRjB3BClGSvGLve+KKRSxU9I/6GHMmW2N9/t1IJau4QjqlnxggDqlu
+r6VwCBVPuOUbR0K0pXdiILcFrGYg3OhYnjqti/IHPvSmmILzNIDknEsuTlw4lwlE
+Oqhul9wh6EJH/JTCydMg/0twrn8CHTcNzX6VDpXb16F+d65HnOo5jeabAOjWVlCb
+b42qOUsUqSckQs8Ag7aLnhhq7s99q19hZoC9BjS9dFEB2iPzq4wa/kO4gsVrLSBv
+pAQMz3/DnLV/G81l2RnMotJi2s9S4t18uQTjsyPID5Uh0uT1rwGxDGhkCp7PhrsD
+RXjsLdjVuANlkHYs
+-----END CERTIFICATE-----
diff --git a/tests/remote-e2e/mtls/server/server.crt b/tests/remote-e2e/mtls/server/server.crt
new file mode 100644
index 0000000..3a469f3
--- /dev/null
+++ b/tests/remote-e2e/mtls/server/server.crt
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFjCCAv6gAwIBAgIURSZT/HoeM2opaXy9omTS8HnKyp8wDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPc3ctbXRscy10ZXN0LWNhMB4XDTI2MDkxNTA2NDczNVoX
+DTM2MDkxMjA2NTIzNVowDjEMMAoGA1UEAwwDb2FwMIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEA6DZX6kJO02xtQHrqgPYwXiaJGEKwAJ0HV3mCBb1bM2t9
+HsLsWGRBp/nBPxWN3Ug7rP+7FQ2TUm2ERv40gzk9AsY+WnJKIq5TN5Oy04I6T5wj
+b30fDloMEVAOAq9TUzWGhNwThWJNXyznkVNqo+sz74rzdte+pDu5Qr1kjjmXyTOR
+JfSQHsEmNCYbND4f8lPHYHo2+cwqpiYFBP0bpJuQgZH2WVZ2oLgcAiSRh5Q5VVdb
+xuuafDDNOiC7Q55cwjgRhfNVIDhJWQ+VQt16HbE0moVg1WG8B2tTkTmJFS0Aj8Pk
+BITqpCbelQ1k6G4bFZpCiTmckc8uAT2PfdXwowdORuYHobEXjGaUbmGYWAwQ0kcy
+2ftUZJhGAcmpIcqGlHLxa+ks8pwGw/PATXjv45sxTmOT/Vodic9JsNTqaVZ6TvJQ
+huiYIR8ElR3fsA2mUOwwT+9t69uuCetZBn43LaEI+H9Uvso8W/vM9TUcKYcByfhv
+LfUJ3IuVYF0DW5eOHSer96ElYzaY0G0hbTjBZDx+4XppqQ92aBuDFpdP1Wday8R4
+Ea5WXO4TxJzoN4SdBGrFYPIMs+Z4BQPGIfuHnlSdiRozBQoqBlUBATZdypuM7IrW
+WJayKlQIjmrDFyCGHFGrnl2Vr4601YGkDhq1g4ZxRuPHvNxcyTQb9zt9iVE0MPcC
+AwEAAaNgMF4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwEGCCsGAQUFBwMCMB8GA1UdEQQYMBaCA29hcIIJbG9jYWxob3N0
+hwR/AAABMA0GCSqGSIb3DQEBCwUAA4ICAQBw6ss183bQw8oQTCUPsZY4AV+iETpw
+DIy07zISAHVenNuMrfgg86T6hWSg7vh10Bub6fDxsMHrLXskA9fbzv5zyPjgosyq
+x7CMuGl+3fXdxxebdvc17W65wqXlWXHFzHPCo/g8oteF2PvaywVEaoNCSHXYWHve
+fyMuNnVQHz8tcpyxfWNruzZ1J3DTroEb43acctcmftXolM3eGoPX8d5Z/nJHBYGU
+cpTmO51WeSsZQcJmo4N+fxpwCLY/HzGUQblWkigaEAeFCcwkZiX38k9eQ7c06/FF
+d7gp4oLt4GkaRDg9ogFtUZWQ6RxPhYhkLS8xR+OHwOa9ey1EO4DeYRuTEKP97ipS
+Bzhd061V3GZjLQTffcJn/mthStzz53H+EP8790c+Jb+I/v5yPvM4PqxAtsdBM23F
+Gl0WAsatXprtr2VmcsHd3aGCKY+4m8MSmHZcK/6SVbqoMeIfU3NwCk4K+qWDgbcO
+ukMrwM5Bj1TZ4IswIbdP74Xh2PPUZIsKBdKwqYf3FYBUtTi0nbB0aeeMJa+OMR0u
+eKQaP5GunVqGwWKMEwDszGufq0D8ZeIhBUulW4iZqjEhOyLB9+zClI9kK10XMWtt
+GRsro5kMZvRtuPDJZtoaxWL6sKBcxnmsYTmuO4X5YcE1jcGCULuvPGIod4MkO2aV
+RTmetnYpRqjopg==
+-----END CERTIFICATE-----
diff --git a/tests/remote-e2e/mtls/test.ts b/tests/remote-e2e/mtls/test.ts
new file mode 100644
index 0000000..0c21c7f
--- /dev/null
+++ b/tests/remote-e2e/mtls/test.ts
@@ -0,0 +1,252 @@
+/*!
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/* eslint-env jest */
+
+import axios from 'axios';
+import waitForExpect from 'wait-for-expect';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as tls from 'tls';
+import { DockerComposeEnvironment, StartedDockerComposeEnvironment, Wait } from 'testcontainers';
+
+const rootDir = path.resolve(__dirname);
+const serverUrl = 'http://localhost:5011';
+const oapTlsPort = 11811;
+const oapGraphqlUrl = 'http://localhost:12801/graphql';
+const serviceName = 'nodejs-mtls-e2e';
+
+/** OAP Duration with MINUTE step: `yyyy-MM-dd HHmm` in UTC. */
+function formatUtcMinute(date: Date): string {
+ const pad = (n: number) => String(n).padStart(2, '0');
+ return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())} ${pad(
+ date.getUTCHours(),
+ )}${pad(date.getUTCMinutes())}`;
+}
+
+/**
+ * True only when the TLS failure clearly means the peer required a client
+ * certificate. Hostname / CA / cipher failures must NOT match.
+ */
+function isClientCertificateRequiredError(error: Error & { code?: string }): boolean {
+ const code = error.code ?? '';
+ const message = error.message;
+
+ if (
+ code === 'ERR_TLS_CERT_ALTNAME_INVALID' ||
+ code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE' ||
+ code === 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY' ||
+ code === 'CERT_HAS_EXPIRED' ||
+ code === 'DEPTH_ZERO_SELF_SIGNED_CERT' ||
+ code === 'ERR_SSL_WRONG_VERSION_NUMBER' ||
+ /altname|hostname|cipher|protocol version|unable to verify|self.signed/i.test(message)
+ ) {
+ return false;
+ }
+
+ return (
+ code === 'ERR_SSL_PEER_CERTIFICATE_REQUIRED' ||
+ code === 'ERR_SSL_SSLV3_ALERT_CERTIFICATE_REQUIRED' ||
+ code === 'ERR_SSL_TLSV13_ALERT_CERTIFICATE_REQUIRED' ||
+ /certificate.?required|peer did not return a certificate|client.?certificate.?required/i.test(message)
+ );
+}
+
+function assertOapRequiresClientCertificate(): Promise<void> {
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ let handshakeCompleted = false;
+ const socket = tls.connect({
+ host: '127.0.0.1',
+ port: oapTlsPort,
+ ca: fs.readFileSync(path.join(rootDir, 'server', 'ca.crt')),
+ servername: 'oap',
+ rejectUnauthorized: true,
+ // Intentionally omit key/cert — OAP must reject this connection.
+ });
+ const timer = setTimeout(() => {
+ socket.destroy();
+ finish(new Error('OAP accepted a TLS connection without a client certificate'));
+ }, 10000);
+
+ function finish(error?: Error): void {
+ if (settled) {
+ return;
+ }
+ settled = true;
+ clearTimeout(timer);
+ if (error) {
+ reject(error);
+ } else {
+ resolve();
+ }
+ }
+
+ socket.once('secureConnect', () => {
+ // TLS 1.3 may fire secureConnect before the server sends a
+ // certificate-required alert. Do not treat this as acceptance —
+ // only an explicit client-certificate-required error counts.
+ handshakeCompleted = true;
+ });
+ socket.once('error', (error: Error & { code?: string }) => {
+ const code = error.code ?? '';
+ const message = error.message;
+
+ const isNetworkError = /^(ECONNREFUSED|ETIMEDOUT|ECONNRESET|EPIPE|ENOTFOUND|EHOSTUNREACH)$/.test(code);
+ if (isNetworkError) {
+ finish(new Error(`Infra failure reaching OAP TLS port (${code}): ${message}`));
+ return;
+ }
+
+ if (isClientCertificateRequiredError(error)) {
+ finish();
+ return;
+ }
+
+ finish(new Error(`TLS error is not a client-certificate rejection (code=${code}): ${message}`));
+ });
+ socket.once('close', () => {
+ if (settled) {
+ return;
+ }
+ // A bare post-handshake close is not proof of mTLS enforcement — OAP
+ // could close for other reasons. Require an explicit certificate-
+ // required alert/error (handled above).
+ finish(
+ new Error(
+ handshakeCompleted
+ ? 'OAP closed after TLS handshake without a client-certificate-required alert'
+ : 'OAP closed the connection without reporting a TLS handshake failure',
+ ),
+ );
+ });
+ });
+}
+
+async function queryGraphql<T>(query: string, variables?: Record<string, unknown>): Promise<T> {
+ const response = await axios.post(oapGraphqlUrl, { query, variables });
+ expect(response.status).toBe(200);
+ expect(response.data.errors).toBeUndefined();
+ return response.data.data as T;
+}
+
+describe('remote-e2e mTLS with real OAP', () => {
+ let compose: StartedDockerComposeEnvironment | undefined;
+
+ beforeAll(async () => {
+ compose = await new DockerComposeEnvironment(rootDir, 'docker-compose.yml')
+ .withWaitStrategy('server-1', Wait.forHealthCheck())
+ .up();
+ });
+
+ afterAll(async () => {
+ await compose?.down();
+ });
+
+ it('rejects a TLS connection without a client certificate', async () => {
+ if (!compose) {
+ throw new Error('Docker Compose environment failed to start');
+ }
+
+ await assertOapRequiresClientCertificate();
+ }, 300000);
+
+ it('reports a trace to OAP through a mutually authenticated TLS channel', async () => {
+ if (!compose) {
+ throw new Error('Docker Compose environment failed to start');
+ }
+
+ await waitForExpect(async () => {
+ expect((await axios.get(`${serverUrl}/ping`)).status).toBe(200);
+ });
+
+ // Generate a real entry span via the instrumented HTTP server, then flush.
+ // flush() is best-effort and swallows report failures, so HTTP 200 alone
+ // is not proof — we assert persistence via OAP queryTraces below.
+ await axios.get(`${serverUrl}/ping`);
+
+ const flush = await axios.get(`${serverUrl}/flush`);
+ expect(flush.status).toBe(200);
+ expect(flush.data).toBe('flushed');
+
+ // Management gRPC over mTLS: service registration visible in metadata.
+ let serviceId = '';
+ await waitForExpect(
+ async () => {
+ const data = await queryGraphql<{ listServices: Array<{ id: string; name: string }> }>(
+ '{ listServices { id name } }',
+ );
+ const service = data.listServices.find((item) => item.name === serviceName);
+ expect(service).toBeDefined();
+ serviceId = service!.id;
+ },
+ 120000,
+ 3000,
+ );
+
+ // TraceSegmentReportService client-streaming over mTLS: segments must be
+ // queryable. queryTraces (v2) is supported on BanyanDB (same API Java
+ // e2e uses via `swctl tv2 ls`); queryBasicTracesByName is not.
+ const now = Date.now();
+ const start = formatUtcMinute(new Date(now - 60 * 60 * 1000));
+ const end = formatUtcMinute(new Date(now + 5 * 60 * 1000));
+
+ await waitForExpect(
+ async () => {
+ const data = await queryGraphql<{
+ queryTraces: { traces: Array<{ spans: Array<{ serviceCode: string; endpointName: string }> }> };
+ }>(
+ `query($serviceId: ID!, $start: String!, $end: String!) {
+ queryTraces(condition: {
+ serviceId: $serviceId
+ queryDuration: { start: $start, end: $end, step: MINUTE }
+ traceState: ALL
+ queryOrder: BY_START_TIME
+ paging: { pageNum: 1, pageSize: 20 }
+ }) {
+ traces {
+ spans {
+ serviceCode
+ endpointName
+ }
+ }
+ }
+ }`,
+ { serviceId, start, end },
+ );
+
+ const spans: Array<{ serviceCode: string; endpointName: string }> = [];
+ for (const trace of data.queryTraces.traces) {
+ spans.push(...trace.spans);
+ }
+ expect(spans.length).toBeGreaterThan(0);
+ expect(spans).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ serviceCode: serviceName,
+ endpointName: expect.stringMatching(/\/ping/),
+ }),
+ ]),
+ );
+ },
+ 120000,
+ 3000,
+ );
+ }, 300000);
+});
diff --git a/tests/remote/GRPCChannelManager.test.ts b/tests/remote/GRPCChannelManager.test.ts
index cf17f37..4473199 100644
--- a/tests/remote/GRPCChannelManager.test.ts
+++ b/tests/remote/GRPCChannelManager.test.ts
@@ -59,6 +59,10 @@ function installChannelMock(): void {
describe('GRPCChannelManager (native grpc-js multi-backend failover)', () => {
const originalCollector = config.collectorAddress;
const originalSecure = config.secure;
+ const originalTrustedCaPath = config.sslTrustedCaPath;
+ const originalKeyPath = config.sslKeyPath;
+ const originalCertChainPath = config.sslCertChainPath;
+ const originalTargetNameOverride = config.sslTargetNameOverride;
beforeEach(() => {
jest.clearAllMocks();
@@ -74,6 +78,10 @@ describe('GRPCChannelManager (native grpc-js multi-backend failover)', () => {
afterEach(() => {
config.collectorAddress = originalCollector;
config.secure = originalSecure;
+ config.sslTrustedCaPath = originalTrustedCaPath;
+ config.sslKeyPath = originalKeyPath;
+ config.sslCertChainPath = originalCertChainPath;
+ config.sslTargetNameOverride = originalTargetNameOverride;
jest.restoreAllMocks();
});
diff --git a/tests/remote/TLSChannelBuilder.test.ts b/tests/remote/TLSChannelBuilder.test.ts
new file mode 100644
index 0000000..d7d518e
--- /dev/null
+++ b/tests/remote/TLSChannelBuilder.test.ts
@@ -0,0 +1,174 @@
+/*!
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/* eslint-env jest */
+
+jest.mock('fs', () => ({
+ ...jest.requireActual('fs'),
+ readFileSync: jest.fn(),
+}));
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as grpc from '@grpc/grpc-js';
+import config from '../../src/config/AgentConfig';
+import TLSChannelBuilder from '../../src/agent/core/remote/TLSChannelBuilder';
+import { ChannelBuildContext } from '../../src/agent/core/remote/ChannelBuilder';
+
+describe('TLSChannelBuilder', () => {
+ const originalSecure = config.secure;
+ const originalTrustedCaPath = config.sslTrustedCaPath;
+ const originalKeyPath = config.sslKeyPath;
+ const originalCertChainPath = config.sslCertChainPath;
+ const originalTargetNameOverride = config.sslTargetNameOverride;
+ const credentials = {} as grpc.ChannelCredentials;
+ let createSsl: jest.SpyInstance;
+ const readFileSync = fs.readFileSync as jest.Mock;
+
+ beforeEach(() => {
+ config.secure = false;
+ delete config.sslTrustedCaPath;
+ delete config.sslKeyPath;
+ delete config.sslCertChainPath;
+ delete config.sslTargetNameOverride;
+ createSsl = jest.spyOn(grpc.credentials, 'createSsl').mockReturnValue(credentials);
+ readFileSync.mockReset();
+ });
+
+ afterEach(() => {
+ config.secure = originalSecure;
+ config.sslTrustedCaPath = originalTrustedCaPath;
+ config.sslKeyPath = originalKeyPath;
+ config.sslCertChainPath = originalCertChainPath;
+ config.sslTargetNameOverride = originalTargetNameOverride;
+ jest.restoreAllMocks();
+ });
+
+ function context(): ChannelBuildContext {
+ return {
+ credentials: {} as grpc.ChannelCredentials,
+ options: {},
+ };
+ }
+
+ it('leaves insecure credentials unchanged when TLS is disabled', () => {
+ const input = context();
+
+ expect(new TLSChannelBuilder().build(input)).toBe(input);
+ expect(createSsl).not.toHaveBeenCalled();
+ expect(readFileSync).not.toHaveBeenCalled();
+ });
+
+ it('rejects TLS material when secure is disabled', () => {
+ config.sslKeyPath = 'client.key';
+
+ expect(() => new TLSChannelBuilder().build(context())).toThrow('secure=true');
+ expect(createSsl).not.toHaveBeenCalled();
+ });
+
+ it('uses system trust when secure TLS has no custom CA', () => {
+ config.secure = true;
+
+ new TLSChannelBuilder().build(context());
+
+ expect(createSsl).toHaveBeenCalledWith(undefined, undefined, undefined);
+ expect(readFileSync).not.toHaveBeenCalled();
+ });
+
+ it('loads a custom CA for one-way TLS', () => {
+ config.secure = true;
+ config.sslTrustedCaPath = 'certs/ca.crt';
+ const ca = Buffer.from('ca');
+ readFileSync.mockReturnValue(ca);
+
+ new TLSChannelBuilder().build(context());
+
+ expect(readFileSync).toHaveBeenCalledWith(path.resolve('certs/ca.crt'));
+ expect(createSsl).toHaveBeenCalledWith(ca, undefined, undefined);
+ });
+
+ it('loads CA, client key, and client certificate chain for mTLS', () => {
+ config.secure = true;
+ config.sslTrustedCaPath = 'certs/ca.crt';
+ config.sslKeyPath = 'certs/client.key';
+ config.sslCertChainPath = 'certs/client.crt';
+ const ca = Buffer.from('ca');
+ const key = Buffer.from('key');
+ const certChain = Buffer.from('cert-chain');
+ readFileSync.mockImplementation((filePath: string) => {
+ const files: Record<string, Buffer> = {
+ [path.resolve('certs/ca.crt')]: ca,
+ [path.resolve('certs/client.key')]: key,
+ [path.resolve('certs/client.crt')]: certChain,
+ };
+ return files[filePath]!;
+ });
+
+ new TLSChannelBuilder().build(context());
+
+ expect(createSsl).toHaveBeenCalledWith(ca, key, certChain);
+ expect(readFileSync).toHaveBeenCalledTimes(3);
+ });
+
+ it('requires both client key and certificate chain for mTLS', () => {
+ config.secure = true;
+ config.sslKeyPath = 'certs/client.key';
+
+ expect(() => new TLSChannelBuilder().build(context())).toThrow(
+ 'Both sslKeyPath and sslCertChainPath must be configured',
+ );
+ expect(readFileSync).not.toHaveBeenCalled();
+ expect(createSsl).not.toHaveBeenCalled();
+ });
+
+ it('fails closed when configured certificate material cannot be read', () => {
+ config.secure = true;
+ config.sslKeyPath = 'certs/client.key';
+ config.sslCertChainPath = 'certs/client.crt';
+ readFileSync.mockImplementation(() => {
+ throw new Error('ENOENT');
+ });
+
+ expect(() => new TLSChannelBuilder().build(context())).toThrow('Failed to read the client private key');
+ expect(createSsl).not.toHaveBeenCalled();
+ });
+
+ it('sets grpc.ssl_target_name_override when configured', () => {
+ config.secure = true;
+ config.sslTrustedCaPath = 'certs/ca.crt';
+ config.sslTargetNameOverride = 'oap.svc.cluster.local';
+ const ca = Buffer.from('ca');
+ readFileSync.mockReturnValue(ca);
+
+ const result = new TLSChannelBuilder().build(context());
+
+ expect(result.options['grpc.ssl_target_name_override']).toBe('oap.svc.cluster.local');
+ expect(result.options['grpc.default_authority']).toBe('oap.svc.cluster.local');
+ expect(createSsl).toHaveBeenCalledWith(ca, undefined, undefined);
+ });
+
+ it('skips hostname override when not configured', () => {
+ config.secure = true;
+
+ const result = new TLSChannelBuilder().build(context());
+
+ expect(result.options['grpc.ssl_target_name_override']).toBeUndefined();
+ expect(result.options['grpc.default_authority']).toBeUndefined();
+ });
+});