You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by zh...@apache.org on 2026/05/25 16:22:38 UTC
(dubbo-kubernetes) branch master updated: Added multi-cluster functionality (#911)
This is an automated email from the ASF dual-hosted git repository.
mfordjody pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 70ad23ff Added multi-cluster functionality (#911)
70ad23ff is described below
commit 70ad23ff274c66a3bc53ebf8981204515d3edf5b
Author: mfordjody <11...@qq.com>
AuthorDate: Tue May 26 00:22:33 2026 +0800
Added multi-cluster functionality (#911)
---
.github/workflows/release.yaml | 127 ++++-
dubboctl/cmd/multicluster.go | 425 +++++++++++++++++
dubboctl/cmd/multicluster_test.go | 91 ++++
dubboctl/cmd/root.go | 1 +
.../pkg/bootstrap/gateway_multicluster.go | 48 ++
dubbod/discovery/pkg/bootstrap/gui.go | 2 +-
.../pkg/bootstrap/proxyless_grpc_controller.go | 123 ++++-
.../bootstrap/proxyless_grpc_controller_test.go | 60 ++-
dubbod/discovery/pkg/bootstrap/server.go | 4 +-
.../config/kube/gateway/deployment_controller.go | 74 ++-
.../kube/gateway/deployment_controller_test.go | 69 +++
dubbod/discovery/pkg/features/dubbo.go | 3 +
.../pkg/xds/endpoints/endpoint_builder.go | 47 +-
.../pkg/xds/endpoints/endpoint_builder_test.go | 68 +++
.../charts/dubbod/files/kube-gateway.yaml | 3 +
.../charts/dubbod/templates/deployment.yaml | 22 +
.../charts/dubbod/templates/eastwest-gateway.yaml | 57 +++
.../charts/dubbod/templates/service.yaml | 38 ++
dubboinstaller/charts/dubbod/values.yaml | 17 +
dubbooperator/pkg/apis/proto/values_types.proto | 46 ++
dubbooperator/pkg/apis/values_types.pb.go | 521 +++++++++++++++++++--
dubbooperator/pkg/render/manifest_test.go | 142 ++++++
pkg/kube/inject/proxyless_test.go | 59 +++
pkg/kube/inject/webhook.go | 4 +
pkg/kube/multicluster/cluster.go | 38 ++
pkg/kube/multicluster/clusterstore.go | 47 +-
pkg/kube/multicluster/eastwest.go | 55 +++
pkg/kube/multicluster/eastwest_test.go | 35 ++
pkg/kube/multicluster/secretcontroller.go | 169 ++++++-
pkg/kube/multicluster/secretcontroller_test.go | 178 +++++++
samples/multicluster/README.md | 99 ++++
.../multicluster/eastwest-nginx-httproute.yaml | 30 +-
tests/e2e/multicluster-primary-remote.sh | 122 +++++
tools/release_notes/generate_release_notes.py | 401 ++++++++++++++++
tools/release_notes/generate_release_notes_test.py | 86 ++++
35 files changed, 3191 insertions(+), 120 deletions(-)
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 064f9e2d..ae73f194 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -18,7 +18,8 @@ name: Dubbo Kubernetes Release
on:
push:
tags:
- - '[0-9]+.[0-9]+.[0-9]+'
+ - '[0-9]*.[0-9]*.[0-9]*'
+ - 'v[0-9]*.[0-9]*.[0-9]*'
permissions:
contents: write
@@ -31,41 +32,117 @@ jobs:
upload_url: ${{ steps.create_release.outputs.upload_url }}
steps:
- - name: Check if Release Exists
- id: check
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Generate Release Notes
+ run: |
+ python3 tools/release_notes/generate_release_notes.py \
+ --tag "${GITHUB_REF_NAME}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --release-body release-notes.md
+
+ - name: Create or Update GitHub Release
+ id: create_release
uses: actions/github-script@v7
with:
- result-encoding: string
script: |
+ const fs = require('fs');
const tag = context.ref.replace('refs/tags/', '');
+ const body = fs.readFileSync('release-notes.md', 'utf8');
+ let release;
try {
- await github.rest.repos.getReleaseByTag({
+ release = await github.rest.repos.getReleaseByTag({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ tag,
+ });
+ release = await github.rest.repos.updateRelease({
owner: context.repo.owner,
repo: context.repo.repo,
- tag: tag,
+ release_id: release.data.id,
+ name: tag,
+ body,
+ draft: false,
+ prerelease: true,
});
- return "exists";
} catch (e) {
- if (e.status === 404) return "not_found";
- throw e;
+ if (e.status !== 404) throw e;
+ release = await github.rest.repos.createRelease({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ tag_name: tag,
+ name: tag,
+ body,
+ draft: false,
+ prerelease: true,
+ });
}
+ core.setOutput('upload_url', release.data.upload_url);
- - name: Create GitHub Release
- id: create_release
- if: steps.check.outputs.result == 'not_found'
- uses: actions/create-release@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - name: Upload Release Notes Artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: release-notes
+ path: release-notes.md
+ retention-days: 30
+
+ sync_kdubbo_docs:
+ name: Sync Kdubbo Docs Release Notes
+ needs: create_release
+ runs-on: ubuntu-latest
+ env:
+ KDUBBO_DOCS_TOKEN: ${{ secrets.KDUBBO_DOCS_TOKEN }}
+ KDUBBO_DOCS_REPOSITORY: ${{ vars.KDUBBO_DOCS_REPOSITORY || 'kdubbo/kdubbo.github.io' }}
+
+ steps:
+ - name: Require docs token
+ run: |
+ if [ -z "${KDUBBO_DOCS_TOKEN}" ]; then
+ echo "::error::KDUBBO_DOCS_TOKEN is required to push release notes to ${KDUBBO_DOCS_REPOSITORY}"
+ exit 1
+ fi
+
+ - name: Checkout code
+ uses: actions/checkout@v4
with:
- tag_name: ${{ github.ref_name }}
- release_name: "${{ github.ref_name }}"
- body: |
- Added MeshService features
- Added security infrastructure capabilities
- Added new subcommands to dubboctl
- Fixed known issues
- draft: false
- prerelease: true
+ fetch-depth: 0
+
+ - name: Checkout kdubbo docs
+ uses: actions/checkout@v4
+ with:
+ repository: ${{ env.KDUBBO_DOCS_REPOSITORY }}
+ token: ${{ env.KDUBBO_DOCS_TOKEN }}
+ path: kdubbo-docs
+
+ - name: Generate kdubbo docs release notes
+ run: |
+ python3 tools/release_notes/generate_release_notes.py \
+ --tag "${GITHUB_REF_NAME}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --docs-root kdubbo-docs/docs \
+ --release-body release-notes.md
+
+ - name: Verify kdubbo docs
+ working-directory: kdubbo-docs
+ run: |
+ python3 -m pip install -r requirements.txt
+ python3 -m mkdocs build --strict --site-dir /tmp/kdubbo-site-build
+
+ - name: Commit kdubbo docs release notes
+ working-directory: kdubbo-docs
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git add docs/latest/release
+ if git diff --cached --quiet; then
+ echo "No kdubbo docs release note changes to commit."
+ exit 0
+ fi
+ git commit -m "docs: add ${GITHUB_REF_NAME} release notes"
+ git push
build_and_upload:
name: Build and Upload Binaries
@@ -180,4 +257,4 @@ jobs:
upload_url: ${{ needs.create_release.outputs.upload_url }}
asset_path: ${{ steps.package_dubbo.outputs.name }}
asset_name: ${{ steps.package_dubbo.outputs.name }}
- asset_content_type: application/octet-stream
\ No newline at end of file
+ asset_content_type: application/octet-stream
diff --git a/dubboctl/cmd/multicluster.go b/dubboctl/cmd/multicluster.go
new file mode 100644
index 00000000..e51bfe95
--- /dev/null
+++ b/dubboctl/cmd/multicluster.go
@@ -0,0 +1,425 @@
+package cmd
+
+import (
+ "encoding/base64"
+ "fmt"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/apache/dubbo-kubernetes/pkg/kube/inject"
+ "github.com/apache/dubbo-kubernetes/pkg/kube/multicluster"
+ "github.com/spf13/cobra"
+ admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/tools/clientcmd"
+ clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+ gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+ "sigs.k8s.io/yaml"
+)
+
+type remoteSecretArgs struct {
+ clusterName string
+ name string
+ namespace string
+ kubeconfig string
+ context string
+}
+
+type remoteManifestArgs struct {
+ clusterName string
+ webhookURL string
+ xdsAddress string
+ caAddress string
+ caBundleFile string
+ revision string
+}
+
+type eastWestGatewayArgs struct {
+ name string
+ namespace string
+ serviceType string
+ port int32
+ targetPort int
+ nodePort int32
+ xdsAddress string
+}
+
+func MulticlusterCmd() *cobra.Command {
+ command := &cobra.Command{
+ Use: "multicluster",
+ Short: "Manage remote clusters for dubbod",
+ }
+ command.AddCommand(createRemoteSecretCmd())
+ command.AddCommand(generateRemoteManifestCmd())
+ command.AddCommand(generateEastWestGatewayCmd())
+ return command
+}
+
+func createRemoteSecretCmd() *cobra.Command {
+ args := &remoteSecretArgs{
+ namespace: "dubbo-system",
+ }
+ command := &cobra.Command{
+ Use: "create-remote-secret",
+ Short: "Generate a remote-cluster Secret manifest",
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ kubeconfig, err := remoteKubeconfig(args.kubeconfig, args.context)
+ if err != nil {
+ return err
+ }
+ secret, err := buildRemoteClusterSecret(*args, kubeconfig)
+ if err != nil {
+ return err
+ }
+ raw, err := yaml.Marshal(secret)
+ if err != nil {
+ return err
+ }
+ _, err = cmd.OutOrStdout().Write(raw)
+ return err
+ },
+ }
+ flags := command.Flags()
+ flags.StringVar(&args.clusterName, "cluster-name", "", "Unique remote cluster name")
+ flags.StringVar(&args.name, "name", "", "Secret name; defaults to dubbo-remote-<cluster-name>")
+ flags.StringVarP(&args.namespace, "namespace", "n", args.namespace, "Namespace where dubbod reads remote cluster Secrets")
+ flags.StringVar(&args.kubeconfig, "kubeconfig", "", "Remote cluster kubeconfig path")
+ flags.StringVar(&args.context, "context", "", "Remote cluster kubeconfig context")
+ _ = command.MarkFlagRequired("cluster-name")
+ return command
+}
+
+func generateRemoteManifestCmd() *cobra.Command {
+ args := &remoteManifestArgs{
+ revision: "default",
+ }
+ command := &cobra.Command{
+ Use: "generate-remote-manifest",
+ Short: "Generate remote-cluster injection webhook manifest",
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ caBundle, err := os.ReadFile(args.caBundleFile)
+ if err != nil {
+ return fmt.Errorf("read CA bundle: %w", err)
+ }
+ manifest, err := buildRemoteWebhookManifest(*args, caBundle)
+ if err != nil {
+ return err
+ }
+ raw, err := yaml.Marshal(manifest)
+ if err != nil {
+ return err
+ }
+ _, err = cmd.OutOrStdout().Write(raw)
+ return err
+ },
+ }
+ flags := command.Flags()
+ flags.StringVar(&args.clusterName, "cluster-name", "", "Unique remote cluster name")
+ flags.StringVar(&args.webhookURL, "webhook-url", "", "Externally reachable dubbod webhook base URL")
+ flags.StringVar(&args.xdsAddress, "xds-address", "", "Externally reachable dubbod xDS address")
+ flags.StringVar(&args.caAddress, "ca-address", "", "Externally reachable dubbod CA address")
+ flags.StringVar(&args.caBundleFile, "ca-bundle-file", "", "PEM CA bundle file for webhook TLS verification")
+ flags.StringVar(&args.revision, "revision", args.revision, "Revision label used by the remote injector")
+ for _, flag := range []string{"cluster-name", "webhook-url", "xds-address", "ca-address", "ca-bundle-file"} {
+ _ = command.MarkFlagRequired(flag)
+ }
+ return command
+}
+
+func generateEastWestGatewayCmd() *cobra.Command {
+ args := &eastWestGatewayArgs{
+ name: "dubbod-eastwest-gateway",
+ namespace: "dubbo-system",
+ serviceType: "LoadBalancer",
+ port: 15443,
+ targetPort: inject.ProxylessXServerPort,
+ }
+ command := &cobra.Command{
+ Use: "generate-eastwest-gateway",
+ Short: "Generate an east-west Gateway manifest for a workload cluster",
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ gw, err := buildEastWestGatewayManifest(*args)
+ if err != nil {
+ return err
+ }
+ raw, err := yaml.Marshal(gw)
+ if err != nil {
+ return err
+ }
+ _, err = cmd.OutOrStdout().Write(raw)
+ return err
+ },
+ }
+ flags := command.Flags()
+ flags.StringVar(&args.name, "name", args.name, "Gateway name")
+ flags.StringVarP(&args.namespace, "namespace", "n", args.namespace, "Gateway namespace")
+ flags.StringVar(&args.serviceType, "service-type", args.serviceType, "Managed gateway Service type")
+ flags.Int32Var(&args.port, "port", args.port, "Externally reachable east-west Gateway port")
+ flags.IntVar(&args.targetPort, "target-port", args.targetPort, "Injected xserver target port used by the managed gateway Service")
+ flags.Int32Var(&args.nodePort, "node-port", 0, "Optional managed gateway Service nodePort")
+ flags.StringVar(&args.xdsAddress, "xds-address", "", "ADS address used by the remote dxgate data plane")
+ _ = command.MarkFlagRequired("xds-address")
+ return command
+}
+
+func remoteKubeconfig(path, contextName string) ([]byte, error) {
+ rules := clientcmd.NewDefaultClientConfigLoadingRules()
+ if path != "" {
+ rules.ExplicitPath = path
+ }
+ rawConfig, err := rules.Load()
+ if err != nil {
+ return nil, fmt.Errorf("load kubeconfig: %w", err)
+ }
+ if contextName == "" {
+ contextName = rawConfig.CurrentContext
+ }
+ context, ok := rawConfig.Contexts[contextName]
+ if !ok {
+ return nil, fmt.Errorf("context %q not found", contextName)
+ }
+ minimal := clientcmdapi.Config{
+ Kind: rawConfig.Kind,
+ APIVersion: rawConfig.APIVersion,
+ CurrentContext: contextName,
+ Contexts: map[string]*clientcmdapi.Context{contextName: context.DeepCopy()},
+ Clusters: map[string]*clientcmdapi.Cluster{},
+ AuthInfos: map[string]*clientcmdapi.AuthInfo{},
+ }
+ if clusterConfig, ok := rawConfig.Clusters[context.Cluster]; ok {
+ minimal.Clusters[context.Cluster] = clusterConfig.DeepCopy()
+ }
+ if authInfo, ok := rawConfig.AuthInfos[context.AuthInfo]; ok {
+ minimal.AuthInfos[context.AuthInfo] = authInfo.DeepCopy()
+ }
+ return clientcmd.Write(minimal)
+}
+
+func buildEastWestGatewayManifest(args eastWestGatewayArgs) (*gatewayv1.Gateway, error) {
+ if strings.TrimSpace(args.name) == "" {
+ return nil, fmt.Errorf("name is required")
+ }
+ if strings.TrimSpace(args.namespace) == "" {
+ return nil, fmt.Errorf("namespace is required")
+ }
+ if args.port <= 0 || args.port > 65535 {
+ return nil, fmt.Errorf("port must be between 1 and 65535")
+ }
+ if args.targetPort <= 0 || args.targetPort > 65535 {
+ return nil, fmt.Errorf("target-port must be between 1 and 65535")
+ }
+ if args.nodePort < 0 || args.nodePort > 65535 {
+ return nil, fmt.Errorf("node-port must be between 0 and 65535")
+ }
+ if strings.TrimSpace(args.xdsAddress) == "" {
+ return nil, fmt.Errorf("xds-address is required")
+ }
+ annotations := map[string]string{
+ "gateway.dubbo.apache.org/eastwest": "true",
+ "gateway.dubbo.apache.org/service-type": args.serviceType,
+ "gateway.dubbo.apache.org/target-port": fmt.Sprintf("%d", args.targetPort),
+ "gateway.dubbo.apache.org/xds-address": args.xdsAddress,
+ }
+ if args.nodePort > 0 {
+ annotations["gateway.dubbo.apache.org/node-port"] = fmt.Sprintf("%d", args.nodePort)
+ }
+ return &gatewayv1.Gateway{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: gatewayv1.GroupVersion.String(),
+ Kind: "Gateway",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: args.name,
+ Namespace: args.namespace,
+ Labels: map[string]string{
+ "app": "dubbod-eastwest-gateway",
+ "gateway.dubbo.apache.org/eastwest": "true",
+ },
+ Annotations: annotations,
+ },
+ Spec: gatewayv1.GatewaySpec{
+ GatewayClassName: gatewayv1.ObjectName("dubbo"),
+ Listeners: []gatewayv1.Listener{{
+ Name: gatewayv1.SectionName("http-eastwest"),
+ Protocol: gatewayv1.HTTPProtocolType,
+ Port: gatewayv1.PortNumber(args.port),
+ AllowedRoutes: &gatewayv1.AllowedRoutes{
+ Namespaces: &gatewayv1.RouteNamespaces{
+ From: ptrGatewayFrom(gatewayv1.NamespacesFromAll),
+ },
+ },
+ }},
+ },
+ }, nil
+}
+
+func ptrGatewayFrom(value gatewayv1.FromNamespaces) *gatewayv1.FromNamespaces {
+ return &value
+}
+
+func buildRemoteClusterSecret(args remoteSecretArgs, kubeconfig []byte) (*corev1.Secret, error) {
+ if strings.TrimSpace(args.clusterName) == "" {
+ return nil, fmt.Errorf("cluster-name is required")
+ }
+ if args.name == "" {
+ args.name = "dubbo-remote-" + args.clusterName
+ }
+ return &corev1.Secret{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: "v1",
+ Kind: "Secret",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: args.name,
+ Namespace: args.namespace,
+ Labels: map[string]string{
+ multicluster.RemoteClusterSecretLabel: multicluster.RemoteClusterSecretLabelValue,
+ },
+ },
+ Type: corev1.SecretTypeOpaque,
+ Data: map[string][]byte{
+ multicluster.RemoteClusterSecretClusterNameKey: []byte(args.clusterName),
+ multicluster.RemoteClusterSecretKubeconfigKey: kubeconfig,
+ },
+ }, nil
+}
+
+func buildRemoteWebhookManifest(args remoteManifestArgs, caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+ if strings.TrimSpace(args.clusterName) == "" {
+ return nil, fmt.Errorf("cluster-name is required")
+ }
+ webhookURL, err := remoteWebhookURL(args.webhookURL, args.clusterName, args.xdsAddress, args.caAddress)
+ if err != nil {
+ return nil, err
+ }
+ sideEffects := admissionregistrationv1.SideEffectClassNone
+ failurePolicy := admissionregistrationv1.Fail
+ reinvocationPolicy := admissionregistrationv1.NeverReinvocationPolicy
+ scope := admissionregistrationv1.NamespacedScope
+ clientConfig := admissionregistrationv1.WebhookClientConfig{
+ URL: &webhookURL,
+ CABundle: caBundle,
+ }
+ return &admissionregistrationv1.MutatingWebhookConfiguration{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: admissionregistrationv1.SchemeGroupVersion.String(),
+ Kind: "MutatingWebhookConfiguration",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "dubbo-remote-proxyless-injector-" + args.clusterName,
+ Labels: map[string]string{
+ "app": "dubbod",
+ "dubbo.apache.org/rev": args.revision,
+ },
+ },
+ Webhooks: []admissionregistrationv1.MutatingWebhook{
+ remoteWebhook("rev.namespace.remote.proxyless-injector.dubbo.apache.org", clientConfig, &failurePolicy, &sideEffects, &reinvocationPolicy, &scope,
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "dubbo.apache.org/rev", Operator: metav1.LabelSelectorOpIn, Values: []string{args.revision}},
+ {Key: "dubbo-injection", Operator: metav1.LabelSelectorOpDoesNotExist},
+ }},
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "proxyless.dubbo.apache.org/inject", Operator: metav1.LabelSelectorOpNotIn, Values: []string{"false"}},
+ }},
+ ),
+ remoteWebhook("rev.object.remote.proxyless-injector.dubbo.apache.org", clientConfig, &failurePolicy, &sideEffects, &reinvocationPolicy, &scope,
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "dubbo.apache.org/rev", Operator: metav1.LabelSelectorOpDoesNotExist},
+ {Key: "dubbo-injection", Operator: metav1.LabelSelectorOpDoesNotExist},
+ }},
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "proxyless.dubbo.apache.org/inject", Operator: metav1.LabelSelectorOpNotIn, Values: []string{"false"}},
+ {Key: "dubbo.apache.org/rev", Operator: metav1.LabelSelectorOpIn, Values: []string{args.revision}},
+ }},
+ ),
+ remoteWebhook("namespace.remote.proxyless-injector.dubbo.apache.org", clientConfig, nil, &sideEffects, &reinvocationPolicy, &scope,
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "dubbo-injection", Operator: metav1.LabelSelectorOpIn, Values: []string{"enabled"}},
+ }},
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "proxyless.dubbo.apache.org/inject", Operator: metav1.LabelSelectorOpNotIn, Values: []string{"false"}},
+ }},
+ ),
+ remoteWebhook("object.remote.proxyless-injector.dubbo.apache.org", clientConfig, nil, &sideEffects, &reinvocationPolicy, &scope,
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "dubbo-injection", Operator: metav1.LabelSelectorOpDoesNotExist},
+ }},
+ &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+ {Key: "proxyless.dubbo.apache.org/inject", Operator: metav1.LabelSelectorOpIn, Values: []string{"true"}},
+ }},
+ ),
+ },
+ }, nil
+}
+
+func remoteWebhook(name string, clientConfig admissionregistrationv1.WebhookClientConfig, failurePolicy *admissionregistrationv1.FailurePolicyType,
+ sideEffects *admissionregistrationv1.SideEffectClass, reinvocationPolicy *admissionregistrationv1.ReinvocationPolicyType,
+ scope *admissionregistrationv1.ScopeType, namespaceSelector, objectSelector *metav1.LabelSelector,
+) admissionregistrationv1.MutatingWebhook {
+ return admissionregistrationv1.MutatingWebhook{
+ Name: name,
+ AdmissionReviewVersions: []string{"v1"},
+ ClientConfig: clientConfig,
+ FailurePolicy: failurePolicy,
+ NamespaceSelector: namespaceSelector,
+ ObjectSelector: objectSelector,
+ ReinvocationPolicy: reinvocationPolicy,
+ Rules: []admissionregistrationv1.RuleWithOperations{{
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ },
+ Rule: admissionregistrationv1.Rule{
+ APIGroups: []string{""},
+ APIVersions: []string{"v1"},
+ Resources: []string{"pods"},
+ Scope: scope,
+ },
+ }, {
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ admissionregistrationv1.Update,
+ },
+ Rule: admissionregistrationv1.Rule{
+ APIGroups: []string{""},
+ APIVersions: []string{"v1"},
+ Resources: []string{"services"},
+ Scope: scope,
+ },
+ }},
+ SideEffects: sideEffects,
+ }
+}
+
+func remoteWebhookURL(rawURL, clusterName, xdsAddress, caAddress string) (string, error) {
+ if strings.TrimSpace(xdsAddress) == "" || strings.TrimSpace(caAddress) == "" {
+ return "", fmt.Errorf("xds-address and ca-address are required")
+ }
+ u, err := url.Parse(rawURL)
+ if err != nil || u.Scheme == "" || u.Host == "" {
+ return "", fmt.Errorf("webhook-url must be an absolute URL")
+ }
+ path := injectPath(clusterName, xdsAddress, caAddress)
+ u.Path = strings.TrimSuffix(u.Path, "/") + path
+ return u.String(), nil
+}
+
+func injectPath(clusterName, xdsAddress, caAddress string) string {
+ segments := []string{
+ "DUBBO_META_CLUSTER_ID", clusterName,
+ "XDS_ADDRESS", xdsAddress,
+ "CA_ADDRESS", caAddress,
+ }
+ for i := 1; i < len(segments); i += 2 {
+ segments[i] = strings.ReplaceAll(segments[i], "/", "--slash--")
+ }
+ return "/inject/" + strings.Join(segments, "/")
+}
+
+func encodedSecretData(secret *corev1.Secret, key string) string {
+ return base64.StdEncoding.EncodeToString(secret.Data[key])
+}
diff --git a/dubboctl/cmd/multicluster_test.go b/dubboctl/cmd/multicluster_test.go
new file mode 100644
index 00000000..1f9bb565
--- /dev/null
+++ b/dubboctl/cmd/multicluster_test.go
@@ -0,0 +1,91 @@
+package cmd
+
+import (
+ "encoding/base64"
+ "strings"
+ "testing"
+)
+
+func TestBuildRemoteClusterSecret(t *testing.T) {
+ secret, err := buildRemoteClusterSecret(remoteSecretArgs{
+ clusterName: "cluster2",
+ namespace: "dubbo-system",
+ }, []byte("kubeconfig"))
+ if err != nil {
+ t.Fatalf("buildRemoteClusterSecret() error = %v", err)
+ }
+ if secret.Name != "dubbo-remote-cluster2" {
+ t.Fatalf("secret name = %q, want dubbo-remote-cluster2", secret.Name)
+ }
+ if got := encodedSecretData(secret, "cluster-name"); got != base64.StdEncoding.EncodeToString([]byte("cluster2")) {
+ t.Fatalf("cluster-name data = %q", got)
+ }
+ if got := encodedSecretData(secret, "kubeconfig"); got != base64.StdEncoding.EncodeToString([]byte("kubeconfig")) {
+ t.Fatalf("kubeconfig data = %q", got)
+ }
+}
+
+func TestRemoteWebhookURLBuildsEnvPath(t *testing.T) {
+ got, err := remoteWebhookURL("https://dubbod.example.com", "cluster2", "xds.example.com:26012", "ca.example.com:26012")
+ if err != nil {
+ t.Fatalf("remoteWebhookURL() error = %v", err)
+ }
+ want := "https://dubbod.example.com/inject/DUBBO_META_CLUSTER_ID/cluster2/XDS_ADDRESS/xds.example.com:26012/CA_ADDRESS/ca.example.com:26012"
+ if got != want {
+ t.Fatalf("remoteWebhookURL() = %q, want %q", got, want)
+ }
+}
+
+func TestBuildRemoteWebhookManifestIncludesCABundle(t *testing.T) {
+ manifest, err := buildRemoteWebhookManifest(remoteManifestArgs{
+ clusterName: "cluster2",
+ webhookURL: "https://dubbod.example.com",
+ xdsAddress: "xds.example.com:26012",
+ caAddress: "ca.example.com:26012",
+ revision: "default",
+ }, []byte("ca-bundle"))
+ if err != nil {
+ t.Fatalf("buildRemoteWebhookManifest() error = %v", err)
+ }
+ if len(manifest.Webhooks) != 4 {
+ t.Fatalf("webhooks = %d, want 4", len(manifest.Webhooks))
+ }
+ for _, webhook := range manifest.Webhooks {
+ if webhook.ClientConfig.URL == nil || !strings.Contains(*webhook.ClientConfig.URL, "/inject/") {
+ t.Fatalf("webhook URL = %v, want remote inject URL", webhook.ClientConfig.URL)
+ }
+ if string(webhook.ClientConfig.CABundle) != "ca-bundle" {
+ t.Fatalf("caBundle = %q, want ca-bundle", string(webhook.ClientConfig.CABundle))
+ }
+ if len(webhook.Rules) != 2 || webhook.Rules[1].Resources[0] != "services" {
+ t.Fatalf("rules = %+v, want pod and service admission rules", webhook.Rules)
+ }
+ }
+}
+
+func TestBuildEastWestGatewayManifest(t *testing.T) {
+ gateway, err := buildEastWestGatewayManifest(eastWestGatewayArgs{
+ name: "dubbod-eastwest-gateway",
+ namespace: "dubbo-system",
+ serviceType: "NodePort",
+ port: 15443,
+ targetPort: 15080,
+ nodePort: 32443,
+ xdsAddress: "http://192.168.15.164:32010",
+ })
+ if err != nil {
+ t.Fatalf("buildEastWestGatewayManifest() error = %v", err)
+ }
+ if gateway.Name != "dubbod-eastwest-gateway" {
+ t.Fatalf("gateway name = %q", gateway.Name)
+ }
+ if gateway.Annotations["gateway.dubbo.apache.org/service-type"] != "NodePort" {
+ t.Fatalf("service type annotation = %q", gateway.Annotations["gateway.dubbo.apache.org/service-type"])
+ }
+ if gateway.Annotations["gateway.dubbo.apache.org/xds-address"] != "http://192.168.15.164:32010" {
+ t.Fatalf("xds address annotation = %q", gateway.Annotations["gateway.dubbo.apache.org/xds-address"])
+ }
+ if len(gateway.Spec.Listeners) != 1 || gateway.Spec.Listeners[0].Port != 15443 {
+ t.Fatalf("listeners = %#v", gateway.Spec.Listeners)
+ }
+}
diff --git a/dubboctl/cmd/root.go b/dubboctl/cmd/root.go
index bff6e66c..7e4059b9 100644
--- a/dubboctl/cmd/root.go
+++ b/dubboctl/cmd/root.go
@@ -147,6 +147,7 @@ func GetRootCmd(args []string) *cobra.Command {
rootCmd.AddCommand(GetCmd(ctx))
rootCmd.AddCommand(AdminCmd(ctx))
rootCmd.AddCommand(DashboardCmd(ctx))
+ rootCmd.AddCommand(MulticlusterCmd())
rootCmd.AddCommand(GuiCmd())
diff --git a/dubbod/discovery/pkg/bootstrap/gateway_multicluster.go b/dubbod/discovery/pkg/bootstrap/gateway_multicluster.go
new file mode 100644
index 00000000..75ceee86
--- /dev/null
+++ b/dubbod/discovery/pkg/bootstrap/gateway_multicluster.go
@@ -0,0 +1,48 @@
+package bootstrap
+
+import (
+ "sync"
+
+ "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/config/kube/gateway"
+ "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/features"
+ "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvr"
+ "github.com/apache/dubbo-kubernetes/pkg/kube/multicluster"
+ "github.com/apache/dubbo-kubernetes/pkg/log"
+)
+
+type remoteGatewayDeploymentController struct {
+ stop chan struct{}
+ closeOnce sync.Once
+}
+
+func (c *remoteGatewayDeploymentController) Close() {
+ c.closeOnce.Do(func() {
+ close(c.stop)
+ })
+}
+
+func (c *remoteGatewayDeploymentController) HasSynced() bool {
+ return true
+}
+
+func (s *Server) initMulticlusterGatewayDeploymentControllers(args *DubboArgs) {
+ if s.multiclusterController == nil || !features.EnableGatewayAPI || !features.EnableGatewayAPIDeploymentController {
+ return
+ }
+ multicluster.BuildMultiClusterComponent(s.multiclusterController, func(remote *multicluster.Cluster) *remoteGatewayDeploymentController {
+ c := &remoteGatewayDeploymentController{stop: make(chan struct{})}
+ if remote == nil || remote.Client == nil || remote.ID == s.clusterID {
+ return c
+ }
+ go func() {
+ if !remote.Client.CrdWatcher().WaitForCRD(gvr.KubernetesGateway, c.stop) {
+ return
+ }
+ controller := gateway.NewDeploymentController(remote.Client, remote.ID, s.environment,
+ s.webhookInfo.getWebhookConfig, s.webhookInfo.addHandler, nil, args.Revision, args.Namespace)
+ log.Infof("starting remote gateway deployment controller for cluster %s", remote.ID)
+ controller.Run(c.stop)
+ }()
+ return c
+ })
+}
diff --git a/dubbod/discovery/pkg/bootstrap/gui.go b/dubbod/discovery/pkg/bootstrap/gui.go
index 1f0af033..6ce57095 100644
--- a/dubbod/discovery/pkg/bootstrap/gui.go
+++ b/dubbod/discovery/pkg/bootstrap/gui.go
@@ -438,7 +438,7 @@ func (s *Server) buildGUIOverview() guiOverview {
CachesSynced: s.cachesSynced(),
ServicesSynced: s.ServiceController().HasSynced(),
ConfigSynced: s.configController != nil && s.configController.HasSynced(),
- ProxylessSynced: s.proxylessGRPCWorkloadController == nil || s.proxylessGRPCWorkloadController.HasSynced(),
+ ProxylessSynced: s.proxylessGRPCWorkloadsSynced(),
InjectorReady: s.kubeClient == nil || s.readinessFlags.InjectorReady.Load(),
ValidationReady: s.kubeClient == nil || s.readinessFlags.configValidationReady.Load(),
},
diff --git a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
index a4abb14c..422e2807 100644
--- a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
+++ b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller.go
@@ -38,6 +38,7 @@ import (
"github.com/apache/dubbo-kubernetes/pkg/kube/controllers"
"github.com/apache/dubbo-kubernetes/pkg/kube/inject"
"github.com/apache/dubbo-kubernetes/pkg/kube/kclient"
+ "github.com/apache/dubbo-kubernetes/pkg/kube/multicluster"
"github.com/apache/dubbo-kubernetes/pkg/log"
pkgmodel "github.com/apache/dubbo-kubernetes/pkg/model"
"github.com/apache/dubbo-kubernetes/pkg/spiffe"
@@ -69,6 +70,7 @@ var proxylessGRPCLog = log.RegisterScope("proxylessgrpc", "proxyless grpc worklo
type proxylessGRPCWorkloadController struct {
server *Server
+ client kubelib.Client
pods kclient.Client[*corev1.Pod]
// managedPods keeps bundle/cert updates from scanning every Pod in the cluster.
managedPods kclient.RawIndexer
@@ -83,13 +85,43 @@ type proxylessGRPCWorkloadController struct {
nextRotation time.Time
}
+type proxylessGRPCClusterController struct {
+ controller *proxylessGRPCWorkloadController
+ stop chan struct{}
+}
+
+func (c *proxylessGRPCClusterController) Close() {
+ if c.stop != nil {
+ close(c.stop)
+ }
+}
+
+func (c *proxylessGRPCClusterController) HasSynced() bool {
+ return c.controller == nil || c.controller.HasSynced()
+}
+
func (s *Server) initProxylessGRPCWorkloads() error {
if s.kubeClient == nil {
return nil
}
- controller := newProxylessGRPCWorkloadController(s)
+ controller := newProxylessGRPCWorkloadController(s, s.kubeClient)
s.proxylessGRPCWorkloadController = controller
+ if s.multiclusterController != nil {
+ s.proxylessGRPCRemoteControllers = multicluster.BuildMultiClusterComponent(s.multiclusterController,
+ func(cluster *multicluster.Cluster) *proxylessGRPCClusterController {
+ if cluster.ID == s.clusterID {
+ return &proxylessGRPCClusterController{}
+ }
+ remote := newProxylessGRPCWorkloadController(s, cluster.Client)
+ wrapped := &proxylessGRPCClusterController{
+ controller: remote,
+ stop: make(chan struct{}),
+ }
+ go remote.Run(wrapped.stop)
+ return wrapped
+ })
+ }
if s.XDSServer != nil {
previous := s.XDSServer.RuntimeConfigUpdate
s.XDSServer.RuntimeConfigUpdate = func(req *discoverymodel.PushRequest) {
@@ -97,6 +129,13 @@ func (s *Server) initProxylessGRPCWorkloads() error {
previous(req)
}
controller.handleRuntimeConfigUpdate(req)
+ if s.proxylessGRPCRemoteControllers != nil {
+ for _, remote := range s.proxylessGRPCRemoteControllers.All() {
+ if remote.controller != nil {
+ remote.controller.handleRuntimeConfigUpdate(req)
+ }
+ }
+ }
}
}
s.addStartFunc(proxylessGRPCControllerName, func(stop <-chan struct{}) error {
@@ -106,11 +145,19 @@ func (s *Server) initProxylessGRPCWorkloads() error {
return nil
}
-func newProxylessGRPCWorkloadController(s *Server) *proxylessGRPCWorkloadController {
+func (s *Server) proxylessGRPCWorkloadsSynced() bool {
+ if s.proxylessGRPCWorkloadController != nil && !s.proxylessGRPCWorkloadController.HasSynced() {
+ return false
+ }
+ return s.proxylessGRPCRemoteControllers == nil || s.proxylessGRPCRemoteControllers.HasSynced()
+}
+
+func newProxylessGRPCWorkloadController(s *Server, client kubelib.Client) *proxylessGRPCWorkloadController {
c := &proxylessGRPCWorkloadController{
server: s,
- pods: kclient.NewFiltered[*corev1.Pod](s.kubeClient, kclient.Filter{
- ObjectFilter: s.kubeClient.ObjectFilter(),
+ client: client,
+ pods: kclient.NewFiltered[*corev1.Pod](client, kclient.Filter{
+ ObjectFilter: client.ObjectFilter(),
}),
rotations: make(map[types.NamespacedName]time.Time),
}
@@ -262,7 +309,7 @@ func (c *proxylessGRPCWorkloadController) reconcile(key types.NamespacedName) er
c.clearRotation(key)
return c.deleteSecretForPod(pod)
}
- secrets := c.server.kubeClient.Kube().CoreV1().Secrets(pod.Namespace)
+ secrets := c.client.Kube().CoreV1().Secrets(pod.Namespace)
secretName := inject.ProxylessGRPCSecretNameForMeta(pod.ObjectMeta)
current, err := secrets.Get(context.Background(), secretName, metav1.GetOptions{})
if err != nil {
@@ -303,7 +350,9 @@ type proxylessGRPCWorkloadContext struct {
podIP string
serviceAccount string
trustDomain string
+ clusterID string
discoveryAddress string
+ caAddress string
}
type proxylessGRPCRuntimeConfig struct {
@@ -454,6 +503,8 @@ func reusableWorkloadCertificate(secret *corev1.Secret, activeRootCert []byte) (
}
func (c *proxylessGRPCWorkloadController) buildWorkloadContext(pod *corev1.Pod) (*proxylessGRPCWorkloadContext, error) {
+ pod = c.latestPodForWorkloadContext(pod)
+
trustDomain := constants.DefaultClusterLocalDomain
if meshCfg := c.server.environment.Mesh(); meshCfg != nil && meshCfg.GetTrustDomain() != "" {
trustDomain = meshCfg.GetTrustDomain()
@@ -476,6 +527,12 @@ func (c *proxylessGRPCWorkloadController) buildWorkloadContext(pod *corev1.Pod)
if podIP == "" {
podIP = "0.0.0.0"
}
+ clusterID := podEnvValue(pod, "DUBBO_META_CLUSTER_ID", constants.DefaultClusterName)
+ discoveryAddress := podEnvValue(pod, inject.ProxylessXDSAddressEnvName, proxyConfig.GetDiscoveryAddress())
+ caAddress := podEnvValue(pod, "CA_ADDRESS", discoveryAddress)
+ proxyConfigCopy := *proxyConfig
+ proxyConfig = &proxyConfigCopy
+ proxyConfig.DiscoveryAddress = discoveryAddress
nodeID := strings.Join([]string{
string(pkgmodel.Proxyless),
@@ -490,7 +547,7 @@ func (c *proxylessGRPCWorkloadController) buildWorkloadContext(pod *corev1.Pod)
ProxyConfig: proxyConfig,
Envs: []string{
"DUBBO_META_GENERATOR=grpc",
- "DUBBO_META_CLUSTER_ID=" + constants.DefaultClusterName,
+ "DUBBO_META_CLUSTER_ID=" + clusterID,
"DUBBO_META_NAMESPACE=" + pod.Namespace,
"DUBBO_META_MESH_ID=" + trustDomain,
"TRUST_DOMAIN=" + trustDomain,
@@ -512,10 +569,52 @@ func (c *proxylessGRPCWorkloadController) buildWorkloadContext(pod *corev1.Pod)
podIP: podIP,
serviceAccount: serviceAccount,
trustDomain: trustDomain,
- discoveryAddress: proxyConfig.GetDiscoveryAddress(),
+ clusterID: clusterID,
+ discoveryAddress: discoveryAddress,
+ caAddress: caAddress,
}, nil
}
+func (c *proxylessGRPCWorkloadController) latestPodForWorkloadContext(pod *corev1.Pod) *corev1.Pod {
+ if pod == nil || c.client == nil || hasConcretePodEnv(pod, "DUBBO_META_CLUSTER_ID") || hasConcretePodEnv(pod, inject.ProxylessXDSAddressEnvName) {
+ return pod
+ }
+ latest, err := c.client.Kube().CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+ if err != nil {
+ proxylessGRPCLog.Warnf("failed to refresh pod before building workload context: %s/%s: %v", pod.Namespace, pod.Name, err)
+ return pod
+ }
+ return latest
+}
+
+func hasConcretePodEnv(pod *corev1.Pod, name string) bool {
+ if pod == nil {
+ return false
+ }
+ for _, container := range pod.Spec.Containers {
+ for _, env := range container.Env {
+ if env.Name == name && env.Value != "" && env.ValueFrom == nil {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func podEnvValue(pod *corev1.Pod, name, fallback string) string {
+ if pod == nil {
+ return fallback
+ }
+ for _, container := range pod.Spec.Containers {
+ for _, env := range container.Env {
+ if env.Name == name && env.Value != "" && env.ValueFrom == nil {
+ return env.Value
+ }
+ }
+ }
+ return fallback
+}
+
func buildBootstrapJSON(workload *proxylessGRPCWorkloadContext) ([]byte, error) {
bootstrapCfg, err := grpcxds.GenerateBootstrap(grpcxds.GenerateBootstrapOptions{
Node: workload.node,
@@ -539,7 +638,7 @@ func buildRuntimeConfigJSON(workload *proxylessGRPCWorkloadContext, services []p
"DUBBO_GRPC_XDS_CREDENTIALS": "true",
"DUBBO_GRPC_XDS_RESOLVER": "xds:///",
"DUBBO_META_GENERATOR": "grpc",
- "DUBBO_META_CLUSTER_ID": constants.DefaultClusterName,
+ "DUBBO_META_CLUSTER_ID": workload.clusterID,
"DUBBO_META_NAMESPACE": workload.podNamespace,
"DUBBO_META_MESH_ID": workload.trustDomain,
"TRUST_DOMAIN": workload.trustDomain,
@@ -547,6 +646,8 @@ func buildRuntimeConfigJSON(workload *proxylessGRPCWorkloadContext, services []p
"POD_NAMESPACE": workload.podNamespace,
"INSTANCE_IP": workload.podIP,
"SERVICE_ACCOUNT": workload.serviceAccount,
+ inject.ProxylessXDSAddressEnvName: workload.discoveryAddress,
+ "CA_ADDRESS": workload.caAddress,
},
Bootstrap: proxylessGRPCBootstrapRuntimeConfig{
Path: inject.ProxylessGRPCBootstrapPath,
@@ -567,7 +668,7 @@ func buildRuntimeConfigJSON(workload *proxylessGRPCWorkloadContext, services []p
PodIP: workload.podIP,
ServiceAccount: workload.serviceAccount,
TrustDomain: workload.trustDomain,
- ClusterID: constants.DefaultClusterName,
+ ClusterID: workload.clusterID,
},
Services: services,
Routes: routes,
@@ -980,13 +1081,13 @@ func shouldManageProxylessGRPCPod(pod *corev1.Pod) bool {
}
func (c *proxylessGRPCWorkloadController) deleteSecret(key types.NamespacedName) error {
- err := c.server.kubeClient.Kube().CoreV1().Secrets(key.Namespace).Delete(context.Background(),
+ err := c.client.Kube().CoreV1().Secrets(key.Namespace).Delete(context.Background(),
inject.ProxylessGRPCSecretName(key.Name), metav1.DeleteOptions{})
return controllers.IgnoreNotFound(err)
}
func (c *proxylessGRPCWorkloadController) deleteSecretForPod(pod *corev1.Pod) error {
- err := c.server.kubeClient.Kube().CoreV1().Secrets(pod.Namespace).Delete(context.Background(),
+ err := c.client.Kube().CoreV1().Secrets(pod.Namespace).Delete(context.Background(),
inject.ProxylessGRPCSecretNameForMeta(pod.ObjectMeta), metav1.DeleteOptions{})
return controllers.IgnoreNotFound(err)
}
diff --git a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller_test.go b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller_test.go
index eda75605..44dc8c9e 100644
--- a/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller_test.go
+++ b/dubbod/discovery/pkg/bootstrap/proxyless_grpc_controller_test.go
@@ -141,7 +141,9 @@ func TestBuildRuntimeConfigJSON(t *testing.T) {
podIP: "10.0.0.1",
serviceAccount: "nginx",
trustDomain: "cluster.local",
- discoveryAddress: "dubbod.dubbo-system.svc:26012",
+ clusterID: "remote",
+ discoveryAddress: "192.168.15.164:32049",
+ caAddress: "192.168.15.164:32049",
}
data, err := buildRuntimeConfigJSON(workload, nil, nil)
@@ -169,6 +171,15 @@ func TestBuildRuntimeConfigJSON(t *testing.T) {
if got.Bootstrap.DiscoveryAddress != workload.discoveryAddress {
t.Fatalf("discoveryAddress = %q, want %q", got.Bootstrap.DiscoveryAddress, workload.discoveryAddress)
}
+ if got.Env["DUBBO_META_CLUSTER_ID"] != workload.clusterID {
+ t.Fatalf("DUBBO_META_CLUSTER_ID = %q, want %q", got.Env["DUBBO_META_CLUSTER_ID"], workload.clusterID)
+ }
+ if got.Env[inject.ProxylessXDSAddressEnvName] != workload.discoveryAddress {
+ t.Fatalf("%s = %q, want %q", inject.ProxylessXDSAddressEnvName, got.Env[inject.ProxylessXDSAddressEnvName], workload.discoveryAddress)
+ }
+ if got.Env["CA_ADDRESS"] != workload.caAddress {
+ t.Fatalf("CA_ADDRESS = %q, want %q", got.Env["CA_ADDRESS"], workload.caAddress)
+ }
if got.Certificates.Provider != grpcxds.FileWatcherCertProviderName {
t.Fatalf("cert provider = %q, want %q", got.Certificates.Provider, grpcxds.FileWatcherCertProviderName)
}
@@ -181,6 +192,53 @@ func TestBuildRuntimeConfigJSON(t *testing.T) {
if got.Workload.PodIP != workload.podIP {
t.Fatalf("podIP = %q, want %q", got.Workload.PodIP, workload.podIP)
}
+ if got.Workload.ClusterID != workload.clusterID {
+ t.Fatalf("clusterId = %q, want %q", got.Workload.ClusterID, workload.clusterID)
+ }
+}
+
+func TestBuildWorkloadContextUsesInjectedRemoteValues(t *testing.T) {
+ controller := &proxylessGRPCWorkloadController{
+ server: &Server{
+ environment: &discoverymodel.Environment{
+ DomainSuffix: constants.DefaultClusterLocalDomain,
+ },
+ },
+ }
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "nginx",
+ Namespace: "app",
+ },
+ Spec: corev1.PodSpec{
+ ServiceAccountName: "nginx",
+ Containers: []corev1.Container{{
+ Name: "app",
+ Env: []corev1.EnvVar{
+ {Name: "DUBBO_META_CLUSTER_ID", Value: "remote"},
+ {Name: inject.ProxylessXDSAddressEnvName, Value: "192.168.15.164:32049"},
+ {Name: "CA_ADDRESS", Value: "192.168.15.164:32049"},
+ },
+ }},
+ },
+ Status: corev1.PodStatus{
+ PodIP: "10.0.0.1",
+ },
+ }
+
+ workload, err := controller.buildWorkloadContext(pod)
+ if err != nil {
+ t.Fatalf("buildWorkloadContext() failed: %v", err)
+ }
+ if workload.clusterID != "remote" {
+ t.Fatalf("clusterID = %q, want remote", workload.clusterID)
+ }
+ if workload.discoveryAddress != "192.168.15.164:32049" {
+ t.Fatalf("discoveryAddress = %q, want 192.168.15.164:32049", workload.discoveryAddress)
+ }
+ if workload.caAddress != "192.168.15.164:32049" {
+ t.Fatalf("caAddress = %q, want 192.168.15.164:32049", workload.caAddress)
+ }
}
func TestBuildRuntimeTrafficConfigCapturesProxylessSecurity(t *testing.T) {
diff --git a/dubbod/discovery/pkg/bootstrap/server.go b/dubbod/discovery/pkg/bootstrap/server.go
index 012cd980..46b0a63e 100644
--- a/dubbod/discovery/pkg/bootstrap/server.go
+++ b/dubbod/discovery/pkg/bootstrap/server.go
@@ -134,6 +134,7 @@ type Server struct {
RWConfigStore model.ConfigStoreController
proxylessGRPCWorkloadController *proxylessGRPCWorkloadController
+ proxylessGRPCRemoteControllers *multicluster.Component[*proxylessGRPCClusterController]
statusManager *status.Manager
}
@@ -668,6 +669,7 @@ func (s *Server) initControllers(args *DubboArgs) error {
if err := s.initConfigController(args); err != nil {
return fmt.Errorf("error initializing config controller: %v", err)
}
+ s.initMulticlusterGatewayDeploymentControllers(args)
if err := s.initServiceControllers(args); err != nil {
return fmt.Errorf("error initializing service controllers: %v", err)
}
@@ -814,7 +816,7 @@ func (s *Server) cachesSynced() bool {
if !s.configController.HasSynced() {
return false
}
- if s.proxylessGRPCWorkloadController != nil && !s.proxylessGRPCWorkloadController.HasSynced() {
+ if !s.proxylessGRPCWorkloadsSynced() {
return false
}
return true
diff --git a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
index 8c2e66a6..c5befbaa 100644
--- a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
+++ b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller.go
@@ -24,6 +24,7 @@ import (
"encoding/json"
"fmt"
"sort"
+ "strconv"
"strings"
appsv1 "k8s.io/api/apps/v1"
@@ -66,6 +67,14 @@ type classInfo struct {
const defaultDxgateGatewayName = "dxgate-gateway"
+const (
+ eastWestGatewayAnnotation = "gateway.dubbo.apache.org/eastwest"
+ serviceTypeAnnotation = "gateway.dubbo.apache.org/service-type"
+ serviceTargetPortAnnotation = "gateway.dubbo.apache.org/target-port"
+ serviceNodePortAnnotation = "gateway.dubbo.apache.org/node-port"
+ xdsAddressAnnotation = "gateway.dubbo.apache.org/xds-address"
+)
+
var builtinClasses = getBuiltinClasses()
var classInfos = getClassInfos()
@@ -332,7 +341,7 @@ func (d *DeploymentController) configureGateway(log *dubbolog.Logger, gw gateway
defaultName := getDefaultName(gw.Name, &gw.Spec, gi.disableNameSuffix)
legacyName := getLegacyDefaultName(gw.Name, &gw.Spec, gi.disableNameSuffix)
- serviceType := gi.defaultServiceType
+ serviceType := serviceTypeForGateway(gw, gi.defaultServiceType)
// Extract service ports from Gateway listeners
ports := extractServicePorts(gw)
@@ -340,7 +349,7 @@ func (d *DeploymentController) configureGateway(log *dubbolog.Logger, gw gateway
log.Infof("gateway has no supported HTTP listeners, skipping dxgate deployment")
return nil
}
- bootstrapConfig, bootstrapConfigHash, err := d.buildDxgateBootstrapConfig(gw.Namespace, defaultName, ports)
+ bootstrapConfig, bootstrapConfigHash, err := d.buildDxgateBootstrapConfig(gw, defaultName, ports)
if err != nil {
log.Errorf("failed building dxgate bootstrap config: %v", err)
return err
@@ -494,14 +503,18 @@ func (d *DeploymentController) domainSuffix() string {
return constants.DefaultClusterLocalDomain
}
-func (d *DeploymentController) buildDxgateBootstrapConfig(namespace, serviceName string, ports []corev1.ServicePort) (string, string, error) {
+func (d *DeploymentController) buildDxgateBootstrapConfig(gw gateway.Gateway, serviceName string, ports []corev1.ServicePort) (string, string, error) {
systemNamespace := d.systemNamespace
if systemNamespace == "" {
systemNamespace = constants.DubboSystemNamespace
}
+ xdsAddress := fmt.Sprintf("http://dubbod.%s.svc:26010", systemNamespace)
+ if gw.Annotations[xdsAddressAnnotation] != "" {
+ xdsAddress = gw.Annotations[xdsAddressAnnotation]
+ }
return buildDxgateBootstrapConfig(
- fmt.Sprintf("http://dubbod.%s.svc:26010", systemNamespace),
- dxgateListenerNames(namespace, serviceName, d.domainSuffix(), ports),
+ xdsAddress,
+ dxgateListenerNames(gw.Namespace, serviceName, d.domainSuffix(), ports),
string(d.clusterID),
d.domainSuffix(),
)
@@ -1038,8 +1051,20 @@ func getLegacyDefaultName(name string, kgw *gateway.GatewaySpec, disableNameSuff
return fmt.Sprintf("%v-%v", name, kgw.GatewayClassName)
}
+func serviceTypeForGateway(gw gateway.Gateway, fallback corev1.ServiceType) corev1.ServiceType {
+ if value := gw.Annotations[serviceTypeAnnotation]; value != "" {
+ switch corev1.ServiceType(value) {
+ case corev1.ServiceTypeClusterIP, corev1.ServiceTypeNodePort, corev1.ServiceTypeLoadBalancer:
+ return corev1.ServiceType(value)
+ }
+ }
+ return fallback
+}
+
func extractServicePorts(gw gateway.Gateway) []corev1.ServicePort {
svcPorts := make([]corev1.ServicePort, 0, len(gw.Spec.Listeners))
+ targetPort := gatewayServiceTargetPort(gw)
+ nodePort := gatewayServiceNodePort(gw)
for i, l := range gw.Spec.Listeners {
if l.Protocol != gateway.HTTPProtocolType {
continue
@@ -1049,16 +1074,49 @@ func extractServicePorts(gw gateway.Gateway) []corev1.ServicePort {
name = fmt.Sprintf("%s-%d", strings.ToLower(string(l.Protocol)), i)
}
appProtocol := strings.ToLower(string(l.Protocol))
- svcPorts = append(svcPorts, corev1.ServicePort{
+ port := corev1.ServicePort{
Name: name,
Port: l.Port,
- TargetPort: intstr.FromString("http"),
+ TargetPort: targetPort,
AppProtocol: &appProtocol,
- })
+ }
+ if nodePort > 0 {
+ port.NodePort = nodePort
+ }
+ svcPorts = append(svcPorts, port)
}
return svcPorts
}
+func gatewayServiceTargetPort(gw gateway.Gateway) intstr.IntOrString {
+ if port, ok := positiveIntAnnotation(gw, serviceTargetPortAnnotation); ok {
+ return intstr.FromInt(port)
+ }
+ if gw.Annotations[eastWestGatewayAnnotation] == "true" {
+ return intstr.FromInt(inject.ProxylessXServerPort)
+ }
+ return intstr.FromString("http")
+}
+
+func gatewayServiceNodePort(gw gateway.Gateway) int32 {
+ port, ok := positiveIntAnnotation(gw, serviceNodePortAnnotation)
+ if !ok {
+ return 0
+ }
+ return int32(port)
+}
+
+func positiveIntAnnotation(gw gateway.Gateway, name string) (int, bool) {
+ if gw.Annotations == nil || gw.Annotations[name] == "" {
+ return 0, false
+ }
+ port, err := strconv.Atoi(gw.Annotations[name])
+ if err != nil || port <= 0 {
+ return 0, false
+ }
+ return port, true
+}
+
// splitYAML splits a YAML document into individual resources
func splitYAML(yamlText string) []string {
out := make([]string, 0)
diff --git a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
index 3cc8b9a8..f9c0a34f 100644
--- a/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
+++ b/dubbod/discovery/pkg/config/kube/gateway/deployment_controller_test.go
@@ -226,6 +226,36 @@ func TestBuildDxgateBootstrapConfig(t *testing.T) {
}
}
+func TestDeploymentControllerBuildDxgateBootstrapConfigUsesXDSAddressAnnotation(t *testing.T) {
+ controller := &DeploymentController{
+ clusterID: "remote",
+ systemNamespace: "dubbo-system",
+ }
+ raw, _, err := controller.buildDxgateBootstrapConfig(gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "eastwest",
+ Namespace: "dubbo-system",
+ Annotations: map[string]string{
+ xdsAddressAnnotation: "http://192.168.15.164:32010",
+ },
+ },
+ }, "dxgate-gateway", []corev1.ServicePort{{Name: "http-eastwest", Port: 15443, TargetPort: intstr.FromInt(15080)}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var cfg dxgateBootstrapConfig
+ if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
+ t.Fatal(err)
+ }
+ if cfg.XDSAddress != "http://192.168.15.164:32010" {
+ t.Fatalf("xdsAddress = %q, want annotation", cfg.XDSAddress)
+ }
+ if cfg.ClusterID != "remote" {
+ t.Fatalf("clusterID = %q, want remote", cfg.ClusterID)
+ }
+}
+
func TestExtractServicePortsTargetsDxgateContainerPorts(t *testing.T) {
gw := gatewayv1.Gateway{
Spec: gatewayv1.GatewaySpec{
@@ -245,6 +275,45 @@ func TestExtractServicePortsTargetsDxgateContainerPorts(t *testing.T) {
}
}
+func TestExtractServicePortsTargetsXServerForEastWestGateway(t *testing.T) {
+ gw := gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ eastWestGatewayAnnotation: "true",
+ serviceNodePortAnnotation: "32443",
+ },
+ },
+ Spec: gatewayv1.GatewaySpec{
+ Listeners: []gatewayv1.Listener{
+ {Name: "http-eastwest", Protocol: gatewayv1.HTTPProtocolType, Port: 15443},
+ },
+ },
+ }
+
+ ports := extractServicePorts(gw)
+ if len(ports) != 1 {
+ t.Fatalf("ports = %d, want 1", len(ports))
+ }
+ if ports[0].TargetPort.IntValue() != 15080 {
+ t.Fatalf("targetPort = %s, want 15080", ports[0].TargetPort.String())
+ }
+ if ports[0].NodePort != 32443 {
+ t.Fatalf("nodePort = %d, want 32443", ports[0].NodePort)
+ }
+}
+
+func TestServiceTypeForGatewayUsesAnnotation(t *testing.T) {
+ gw := gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{serviceTypeAnnotation: "NodePort"},
+ },
+ }
+
+ if got := serviceTypeForGateway(gw, corev1.ServiceTypeLoadBalancer); got != corev1.ServiceTypeNodePort {
+ t.Fatalf("serviceType = %s, want NodePort", got)
+ }
+}
+
func TestGetDefaultNameUsesFixedDxgateGatewayName(t *testing.T) {
spec := &gatewayv1.GatewaySpec{GatewayClassName: "dubbo"}
if got := getDefaultName("httpbin-gateway", spec, false); got != "dxgate-gateway" {
diff --git a/dubbod/discovery/pkg/features/dubbo.go b/dubbod/discovery/pkg/features/dubbo.go
index feb07c16..8dfa92d1 100644
--- a/dubbod/discovery/pkg/features/dubbo.go
+++ b/dubbod/discovery/pkg/features/dubbo.go
@@ -52,6 +52,9 @@ var (
DubbodServiceCustomHost = env.Register("DUBBOD_CUSTOM_HOST", "",
"Custom host name of dubbod that dubbod signs the server cert. "+
"Multiple custom host names are supported, and multiple values are separated by commas.").Get()
+ EastWestGatewayRegistry = env.Register("DUBBO_EASTWEST_GATEWAYS", "",
+ "Comma-separated east-west gateway endpoints in cluster=host:port format. "+
+ "When set, endpoints from another cluster are advertised through that cluster's gateway instead of its pod IPs.").Get()
InjectionWebhookConfigName = env.Register("INJECTION_WEBHOOK_CONFIG_NAME", "dubbo-proxyless-injector",
"Name of the mutatingwebhookconfiguration to patch, if dubboctl is not used.").Get()
ManagedGatewayController = env.Register("DUBBO_GATEWAY_API_CONTROLLER_NAME", "dubbo.apache.org/gateway-controller",
diff --git a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
index fea15978..7c080f73 100644
--- a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
+++ b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder.go
@@ -19,11 +19,14 @@ package endpoints
import (
"fmt"
+ "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/features"
"github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/model"
"github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/networking/util"
+ "github.com/apache/dubbo-kubernetes/pkg/cluster"
"github.com/apache/dubbo-kubernetes/pkg/config/host"
"github.com/apache/dubbo-kubernetes/pkg/config/labels"
"github.com/apache/dubbo-kubernetes/pkg/kube/inject"
+ "github.com/apache/dubbo-kubernetes/pkg/kube/multicluster"
dubbolog "github.com/apache/dubbo-kubernetes/pkg/log"
"github.com/cespare/xxhash/v2"
networking "github.com/kdubbo/api/networking/v1alpha3"
@@ -77,6 +80,14 @@ func (b *EndpointBuilder) ServiceFound() bool {
// BuildClusterLoadAssignment converts the shards for this EndpointBuilder's Service
// into a ClusterLoadAssignment. Used for EDS.
func (b *EndpointBuilder) BuildClusterLoadAssignment(endpointIndex *model.EndpointIndex) *endpoint.ClusterLoadAssignment {
+ gateways, err := multicluster.ParseEastWestGateways(features.EastWestGatewayRegistry)
+ if err != nil {
+ log.Warnf("invalid %s value %q: %v", multicluster.EastWestGatewayEnvName, features.EastWestGatewayRegistry, err)
+ }
+ return b.BuildClusterLoadAssignmentWithGateways(endpointIndex, gateways)
+}
+
+func (b *EndpointBuilder) BuildClusterLoadAssignmentWithGateways(endpointIndex *model.EndpointIndex, gateways map[cluster.ID]multicluster.EastWestGateway) *endpoint.ClusterLoadAssignment {
if !b.ServiceFound() {
return buildEmptyClusterLoadAssignment(b.clusterName)
}
@@ -145,7 +156,7 @@ func (b *EndpointBuilder) BuildClusterLoadAssignment(endpointIndex *model.Endpoi
b.hostname, b.port, svcPort.Name, portNamesList)
}
- for _, eps := range shards.Shards {
+ for shard, eps := range shards.Shards {
for _, ep := range eps {
totalEndpoints++
// Filter by port name
@@ -171,8 +182,7 @@ func (b *EndpointBuilder) BuildClusterLoadAssignment(endpointIndex *model.Endpoi
continue
}
- // Build LbEndpoint
- lbEp := b.buildLbEndpoint(ep)
+ lbEp := b.buildLbEndpointForCluster(ep, shard.Cluster, gateways)
if lbEp == nil {
buildFailedCount++
filteredCount++
@@ -256,12 +266,19 @@ func (b *EndpointBuilder) matchesSubset(epLabels labels.Instance) bool {
return selector.SubsetOf(epLabels)
}
-func (b *EndpointBuilder) buildLbEndpoint(ep *model.DubboEndpoint) *endpoint.LbEndpoint {
+func (b *EndpointBuilder) buildLbEndpointForCluster(ep *model.DubboEndpoint, endpointCluster cluster.ID, gateways map[cluster.ID]multicluster.EastWestGateway) *endpoint.LbEndpoint {
if len(ep.Addresses) == 0 {
return nil
}
- address := util.BuildAddress(ep.Addresses[0], b.endpointPort(ep))
+ endpointAddress := ep.Addresses[0]
+ endpointPort := b.endpointPort(ep)
+ if gateway, ok := b.eastWestGatewayForCluster(endpointCluster, gateways); ok {
+ endpointAddress = gateway.Address
+ endpointPort = gateway.Port
+ }
+
+ address := util.BuildAddress(endpointAddress, endpointPort)
if address == nil {
return nil
}
@@ -292,6 +309,17 @@ func (b *EndpointBuilder) buildLbEndpoint(ep *model.DubboEndpoint) *endpoint.LbE
}
}
+func (b *EndpointBuilder) eastWestGatewayForCluster(endpointCluster cluster.ID, gateways map[cluster.ID]multicluster.EastWestGateway) (multicluster.EastWestGateway, bool) {
+ if endpointCluster == "" || len(gateways) == 0 {
+ return multicluster.EastWestGateway{}, false
+ }
+ if b.proxy != nil && b.proxy.Metadata != nil && b.proxy.Metadata.ClusterID == endpointCluster {
+ return multicluster.EastWestGateway{}, false
+ }
+ gateway, ok := gateways[endpointCluster]
+ return gateway, ok
+}
+
func (b *EndpointBuilder) endpointPort(ep *model.DubboEndpoint) uint32 {
if b.useXServerEndpointPort() {
return inject.ProxylessXServerPort
@@ -348,7 +376,14 @@ func (b *EndpointBuilder) Cacheable() bool {
func (b *EndpointBuilder) Key() any {
// EDS cache expects uint64 key, not string
// Hash the cluster name to uint64 to match the cache type
- return xxhash.Sum64String(b.clusterName)
+ return xxhash.Sum64String(b.clusterName + "|" + string(b.proxyClusterID()) + "|" + features.EastWestGatewayRegistry)
+}
+
+func (b *EndpointBuilder) proxyClusterID() cluster.ID {
+ if b == nil || b.proxy == nil || b.proxy.Metadata == nil {
+ return ""
+ }
+ return b.proxy.Metadata.ClusterID
}
// Type implements model.XdsCacheEntry
diff --git a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
index 21714332..a3f25413 100644
--- a/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
+++ b/dubbod/discovery/pkg/xds/endpoints/endpoint_builder_test.go
@@ -5,6 +5,7 @@ import (
"github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/config/memory"
"github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/model"
+ "github.com/apache/dubbo-kubernetes/pkg/cluster"
"github.com/apache/dubbo-kubernetes/pkg/config"
"github.com/apache/dubbo-kubernetes/pkg/config/host"
"github.com/apache/dubbo-kubernetes/pkg/config/mesh"
@@ -14,6 +15,7 @@ import (
"github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk"
"github.com/apache/dubbo-kubernetes/pkg/kube/inject"
"github.com/apache/dubbo-kubernetes/pkg/kube/krt"
+ "github.com/apache/dubbo-kubernetes/pkg/kube/multicluster"
networking "github.com/kdubbo/api/networking/v1alpha3"
endpoint "github.com/kdubbo/xds-api/endpoint/v1"
)
@@ -63,6 +65,71 @@ func TestBuildClusterLoadAssignmentKeepsAppPortWithoutDUBBOMutual(t *testing.T)
}
}
+func TestBuildClusterLoadAssignmentUsesEastWestGatewayForRemoteShard(t *testing.T) {
+ hostname := host.Name("nginx.app.svc.cluster.local")
+ svc := newEndpointTestService("nginx", "app", string(hostname), 80)
+ push := newEndpointTestPushContext(t, nil, []*model.Service{svc})
+ index := model.NewEndpointIndex(model.DisabledCache{})
+ index.UpdateServiceEndpoints(model.ShardKey{Cluster: cluster.ID("remote")}, string(hostname), "app", []*model.DubboEndpoint{{
+ Addresses: []string{"192.168.219.71"},
+ EndpointPort: 80,
+ ServicePortName: "http",
+ HealthStatus: model.Healthy,
+ }}, false)
+
+ clusterName := model.BuildSubsetKey(model.TrafficDirectionOutbound, "", hostname, 80)
+ proxy := newEndpointTestProxy()
+ proxy.Metadata.ClusterID = cluster.ID("primary")
+ builder := NewEndpointBuilder(clusterName, proxy, push)
+ cla := builder.BuildClusterLoadAssignmentWithGateways(index, map[cluster.ID]multicluster.EastWestGateway{
+ cluster.ID("remote"): {Cluster: cluster.ID("remote"), Address: "192.168.15.155", Port: 15443},
+ })
+
+ if got := firstEndpointAddress(t, cla); got != "192.168.15.155" {
+ t.Fatalf("endpoint address = %q, want east-west gateway", got)
+ }
+ if got := firstEndpointPort(t, cla); got != 15443 {
+ t.Fatalf("endpoint port = %d, want east-west gateway port", got)
+ }
+}
+
+func TestBuildClusterLoadAssignmentKeepsLocalShardPodIPWhenGatewayConfigured(t *testing.T) {
+ hostname := host.Name("nginx.app.svc.cluster.local")
+ svc := newEndpointTestService("nginx", "app", string(hostname), 80)
+ push := newEndpointTestPushContext(t, nil, []*model.Service{svc})
+ index := model.NewEndpointIndex(model.DisabledCache{})
+ index.UpdateServiceEndpoints(model.ShardKey{Cluster: cluster.ID("primary")}, string(hostname), "app", []*model.DubboEndpoint{{
+ Addresses: []string{"10.0.0.1"},
+ EndpointPort: 80,
+ ServicePortName: "http",
+ HealthStatus: model.Healthy,
+ }}, false)
+
+ clusterName := model.BuildSubsetKey(model.TrafficDirectionOutbound, "", hostname, 80)
+ proxy := newEndpointTestProxy()
+ proxy.Metadata.ClusterID = cluster.ID("primary")
+ builder := NewEndpointBuilder(clusterName, proxy, push)
+ cla := builder.BuildClusterLoadAssignmentWithGateways(index, map[cluster.ID]multicluster.EastWestGateway{
+ cluster.ID("primary"): {Cluster: cluster.ID("primary"), Address: "192.168.15.164", Port: 15443},
+ })
+
+ if got := firstEndpointAddress(t, cla); got != "10.0.0.1" {
+ t.Fatalf("endpoint address = %q, want local pod IP", got)
+ }
+ if got := firstEndpointPort(t, cla); got != 80 {
+ t.Fatalf("endpoint port = %d, want local endpoint port", got)
+ }
+}
+
+func firstEndpointAddress(t *testing.T, cla *endpoint.ClusterLoadAssignment) string {
+ t.Helper()
+ localities := cla.GetEndpoints()
+ if len(localities) == 0 || len(localities[0].GetLbEndpoints()) == 0 {
+ t.Fatalf("CLA has no endpoints")
+ }
+ return localities[0].GetLbEndpoints()[0].GetEndpoint().GetAddress().GetSocketAddress().GetAddress()
+}
+
func firstEndpointPort(t *testing.T, cla *endpoint.ClusterLoadAssignment) uint32 {
t.Helper()
localities := cla.GetEndpoints()
@@ -100,6 +167,7 @@ func newEndpointTestProxy() *model.Proxy {
Metadata: &model.NodeMetadata{
Generator: "grpc",
Namespace: "app",
+ ClusterID: cluster.ID("primary"),
},
}
}
diff --git a/dubboinstaller/charts/dubbod/files/kube-gateway.yaml b/dubboinstaller/charts/dubbod/files/kube-gateway.yaml
index e880d51f..de2e38b8 100644
--- a/dubboinstaller/charts/dubbod/files/kube-gateway.yaml
+++ b/dubboinstaller/charts/dubbod/files/kube-gateway.yaml
@@ -162,5 +162,8 @@ spec:
- name: {{ .Name }}
port: {{ .Port }}
targetPort: {{ .TargetPort }}
+{{- if gt (int .NodePort) 0 }}
+ nodePort: {{ .NodePort }}
+{{- end }}
protocol: {{ .Protocol | default "TCP" }}
{{- end }}
diff --git a/dubboinstaller/charts/dubbod/templates/deployment.yaml b/dubboinstaller/charts/dubbod/templates/deployment.yaml
index 075c58cd..5d206f3b 100644
--- a/dubboinstaller/charts/dubbod/templates/deployment.yaml
+++ b/dubboinstaller/charts/dubbod/templates/deployment.yaml
@@ -16,9 +16,23 @@
{{- $defaults := .Values._internal_default_values_not_set | default dict }}
{{- $defaultGlobal := $defaults.global | default dict }}
{{- $defaultGUI := $defaultGlobal.gui | default dict }}
+{{- $defaultMulticluster := $defaultGlobal.multicluster | default dict }}
+{{- $defaultRemoteAccess := $defaultMulticluster.remoteAccess | default dict }}
+{{- $defaultEastWestGateway := $defaultMulticluster.eastWestGateway | default dict }}
{{- $global := .Values.global | default dict }}
{{- $gui := $global.gui | default dict }}
+{{- $multicluster := $global.multicluster | default dict }}
+{{- $remoteAccess := $multicluster.remoteAccess | default dict }}
+{{- $eastWestGateway := $multicluster.eastWestGateway | default dict }}
{{- $guiPort := int (coalesce $gui.port $defaultGUI.port 26080) }}
+{{- $remoteAccessCertificateHosts := $remoteAccess.certificateHosts | default $defaultRemoteAccess.certificateHosts | default (list) }}
+{{- $eastWestGateways := $eastWestGateway.gateways | default $defaultEastWestGateway.gateways | default (list) }}
+{{- $eastWestGatewayEntries := list }}
+{{- $defaultEastWestGatewayPort := int (coalesce $eastWestGateway.port $defaultEastWestGateway.port 15443) }}
+{{- range $gateway := $eastWestGateways }}
+{{- $gatewayPort := int (coalesce $gateway.port $defaultEastWestGatewayPort) }}
+{{- $eastWestGatewayEntries = append $eastWestGatewayEntries (printf "%s=%s:%d" $gateway.clusterName $gateway.address $gatewayPort) }}
+{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -68,6 +82,14 @@ spec:
value: default
- name: DUBBO_CERT_PROVIDER
value: dubbod
+{{- if gt (len $remoteAccessCertificateHosts) 0 }}
+ - name: DUBBOD_CUSTOM_HOST
+ value: {{ join "," $remoteAccessCertificateHosts | quote }}
+{{- end }}
+{{- if gt (len $eastWestGatewayEntries) 0 }}
+ - name: DUBBO_EASTWEST_GATEWAYS
+ value: {{ join "," $eastWestGatewayEntries | quote }}
+{{- end }}
- name: CLUSTER_ID
value: "Kubernetes"
- name: POD_NAME
diff --git a/dubboinstaller/charts/dubbod/templates/eastwest-gateway.yaml b/dubboinstaller/charts/dubbod/templates/eastwest-gateway.yaml
new file mode 100644
index 00000000..7f09d1ea
--- /dev/null
+++ b/dubboinstaller/charts/dubbod/templates/eastwest-gateway.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- $defaults := .Values._internal_default_values_not_set | default dict }}
+{{- $defaultGlobal := $defaults.global | default dict }}
+{{- $defaultMulticluster := $defaultGlobal.multicluster | default dict }}
+{{- $defaultEastWestGateway := $defaultMulticluster.eastWestGateway | default dict }}
+{{- $global := .Values.global | default dict }}
+{{- $multicluster := $global.multicluster | default dict }}
+{{- $eastWestGateway := $multicluster.eastWestGateway | default dict }}
+{{- $eastWestGatewayEnabled := coalesce $eastWestGateway.enabled $defaultEastWestGateway.enabled false }}
+{{- if $eastWestGatewayEnabled }}
+{{- $serviceType := coalesce $eastWestGateway.serviceType $defaultEastWestGateway.serviceType "LoadBalancer" }}
+{{- $port := int (coalesce $eastWestGateway.port $defaultEastWestGateway.port 15443) }}
+{{- $targetPort := int (coalesce $eastWestGateway.targetPort $defaultEastWestGateway.targetPort 15080) }}
+{{- $nodePort := int (coalesce $eastWestGateway.nodePort $defaultEastWestGateway.nodePort 0) }}
+{{- $xdsAddress := coalesce $eastWestGateway.xdsAddress $defaultEastWestGateway.xdsAddress "" }}
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+ name: dubbod-eastwest-gateway
+ namespace: dubbo-system
+ labels:
+ app: dubbod-eastwest-gateway
+ gateway.dubbo.apache.org/eastwest: "true"
+ annotations:
+ gateway.dubbo.apache.org/eastwest: "true"
+ gateway.dubbo.apache.org/service-type: {{ $serviceType | quote }}
+ gateway.dubbo.apache.org/target-port: {{ $targetPort | quote }}
+{{- if $xdsAddress }}
+ gateway.dubbo.apache.org/xds-address: {{ $xdsAddress | quote }}
+{{- end }}
+{{- if gt $nodePort 0 }}
+ gateway.dubbo.apache.org/node-port: {{ $nodePort | quote }}
+{{- end }}
+spec:
+ gatewayClassName: dubbo
+ listeners:
+ - name: http-eastwest
+ protocol: HTTP
+ port: {{ $port }}
+ allowedRoutes:
+ namespaces:
+ from: All
+{{- end }}
diff --git a/dubboinstaller/charts/dubbod/templates/service.yaml b/dubboinstaller/charts/dubbod/templates/service.yaml
index ed11c652..23da4a1b 100644
--- a/dubboinstaller/charts/dubbod/templates/service.yaml
+++ b/dubboinstaller/charts/dubbod/templates/service.yaml
@@ -16,10 +16,19 @@
{{- $defaults := .Values._internal_default_values_not_set | default dict }}
{{- $defaultGlobal := $defaults.global | default dict }}
{{- $defaultGUI := $defaultGlobal.gui | default dict }}
+{{- $defaultMulticluster := $defaultGlobal.multicluster | default dict }}
+{{- $defaultRemoteAccess := $defaultMulticluster.remoteAccess | default dict }}
{{- $global := .Values.global | default dict }}
{{- $gui := $global.gui | default dict }}
+{{- $multicluster := $global.multicluster | default dict }}
+{{- $remoteAccess := $multicluster.remoteAccess | default dict }}
{{- $guiPort := int (coalesce $gui.port $defaultGUI.port 26080) }}
{{- $guiNodePort := int (coalesce $gui.nodePort $defaultGUI.nodePort 30080) }}
+{{- $remoteAccessEnabled := coalesce $remoteAccess.enabled $defaultRemoteAccess.enabled false }}
+{{- $remoteAccessServiceType := coalesce $remoteAccess.serviceType $defaultRemoteAccess.serviceType "LoadBalancer" }}
+{{- $remoteAccessGRPCPort := int (coalesce $remoteAccess.grpcPort $defaultRemoteAccess.grpcPort 26010) }}
+{{- $remoteAccessXDSPort := int (coalesce $remoteAccess.xdsPort $defaultRemoteAccess.xdsPort 26012) }}
+{{- $remoteAccessWebhookPort := int (coalesce $remoteAccess.webhookPort $defaultRemoteAccess.webhookPort 443) }}
apiVersion: v1
kind: Service
metadata:
@@ -66,3 +75,32 @@ spec:
protocol: TCP
selector:
app: dubbod
+{{- if $remoteAccessEnabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: dubbod-remote
+ namespace: dubbo-system
+ labels:
+ app: dubbod
+spec:
+ type: {{ $remoteAccessServiceType }}
+ ports:
+{{- if gt $remoteAccessGRPCPort 0 }}
+ - port: {{ $remoteAccessGRPCPort }}
+ name: grpc-xds
+ targetPort: 26010
+ protocol: TCP
+{{- end }}
+ - port: {{ $remoteAccessXDSPort }}
+ name: tls-xds
+ targetPort: 26012
+ protocol: TCP
+ - port: {{ $remoteAccessWebhookPort }}
+ name: https-webhook
+ targetPort: 26017
+ protocol: TCP
+ selector:
+ app: dubbod
+{{- end }}
diff --git a/dubboinstaller/charts/dubbod/values.yaml b/dubboinstaller/charts/dubbod/values.yaml
index fcb95320..83090e11 100644
--- a/dubboinstaller/charts/dubbod/values.yaml
+++ b/dubboinstaller/charts/dubbod/values.yaml
@@ -27,3 +27,20 @@ _internal_default_values_not_set:
nodePort: 30080
configValidation: true
+
+ multicluster:
+ remoteAccess:
+ enabled: false
+ serviceType: "LoadBalancer"
+ grpcPort: 26010
+ xdsPort: 26012
+ webhookPort: 443
+ certificateHosts: []
+ eastWestGateway:
+ enabled: false
+ serviceType: "LoadBalancer"
+ port: 15443
+ targetPort: 15080
+ nodePort: 0
+ gateways: []
+ xdsAddress: ""
diff --git a/dubbooperator/pkg/apis/proto/values_types.proto b/dubbooperator/pkg/apis/proto/values_types.proto
index 1a980b5e..8aa6c120 100644
--- a/dubbooperator/pkg/apis/proto/values_types.proto
+++ b/dubbooperator/pkg/apis/proto/values_types.proto
@@ -42,6 +42,52 @@ message GlobalConfig {
GUIConfig gui = 3;
bool configValidation = 4;
+
+ MulticlusterConfig multicluster = 5;
+}
+
+message RemoteAccessConfig {
+ bool enabled = 1;
+
+ string serviceType = 2;
+
+ int64 xdsPort = 3;
+
+ int64 webhookPort = 4;
+
+ repeated string certificateHosts = 5;
+
+ int64 grpcPort = 6;
+}
+
+message EastWestGatewayEndpoint {
+ string clusterName = 1;
+
+ string address = 2;
+
+ int64 port = 3;
+}
+
+message EastWestGatewayConfig {
+ bool enabled = 1;
+
+ string serviceType = 2;
+
+ int64 port = 3;
+
+ int64 targetPort = 4;
+
+ int64 nodePort = 5;
+
+ repeated EastWestGatewayEndpoint gateways = 6;
+
+ string xdsAddress = 7;
+}
+
+message MulticlusterConfig {
+ RemoteAccessConfig remoteAccess = 1;
+
+ EastWestGatewayConfig eastWestGateway = 2;
}
message Values {
diff --git a/dubbooperator/pkg/apis/values_types.pb.go b/dubbooperator/pkg/apis/values_types.pb.go
index 907f01c9..0dcc0f62 100644
--- a/dubbooperator/pkg/apis/values_types.pb.go
+++ b/dubbooperator/pkg/apis/values_types.pb.go
@@ -144,10 +144,11 @@ type GlobalConfig struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- Proxy *ProxyConfig `protobuf:"bytes,1,opt,name=proxy,proto3" json:"proxy,omitempty"`
- StatusPort int64 `protobuf:"varint,2,opt,name=statusPort,proto3" json:"statusPort,omitempty"`
- Gui *GUIConfig `protobuf:"bytes,3,opt,name=gui,proto3" json:"gui,omitempty"`
- ConfigValidation bool `protobuf:"varint,4,opt,name=configValidation,proto3" json:"configValidation,omitempty"`
+ Proxy *ProxyConfig `protobuf:"bytes,1,opt,name=proxy,proto3" json:"proxy,omitempty"`
+ StatusPort int64 `protobuf:"varint,2,opt,name=statusPort,proto3" json:"statusPort,omitempty"`
+ Gui *GUIConfig `protobuf:"bytes,3,opt,name=gui,proto3" json:"gui,omitempty"`
+ ConfigValidation bool `protobuf:"varint,4,opt,name=configValidation,proto3" json:"configValidation,omitempty"`
+ Multicluster *MulticlusterConfig `protobuf:"bytes,5,opt,name=multicluster,proto3" json:"multicluster,omitempty"`
}
func (x *GlobalConfig) Reset() {
@@ -210,6 +211,313 @@ func (x *GlobalConfig) GetConfigValidation() bool {
return false
}
+func (x *GlobalConfig) GetMulticluster() *MulticlusterConfig {
+ if x != nil {
+ return x.Multicluster
+ }
+ return nil
+}
+
+type RemoteAccessConfig struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
+ ServiceType string `protobuf:"bytes,2,opt,name=serviceType,proto3" json:"serviceType,omitempty"`
+ XdsPort int64 `protobuf:"varint,3,opt,name=xdsPort,proto3" json:"xdsPort,omitempty"`
+ WebhookPort int64 `protobuf:"varint,4,opt,name=webhookPort,proto3" json:"webhookPort,omitempty"`
+ CertificateHosts []string `protobuf:"bytes,5,rep,name=certificateHosts,proto3" json:"certificateHosts,omitempty"`
+ GrpcPort int64 `protobuf:"varint,6,opt,name=grpcPort,proto3" json:"grpcPort,omitempty"`
+}
+
+func (x *RemoteAccessConfig) Reset() {
+ *x = RemoteAccessConfig{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_values_types_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *RemoteAccessConfig) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RemoteAccessConfig) ProtoMessage() {}
+
+func (x *RemoteAccessConfig) ProtoReflect() protoreflect.Message {
+ mi := &file_values_types_proto_msgTypes[3]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use RemoteAccessConfig.ProtoReflect.Descriptor instead.
+func (*RemoteAccessConfig) Descriptor() ([]byte, []int) {
+ return file_values_types_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *RemoteAccessConfig) GetEnabled() bool {
+ if x != nil {
+ return x.Enabled
+ }
+ return false
+}
+
+func (x *RemoteAccessConfig) GetServiceType() string {
+ if x != nil {
+ return x.ServiceType
+ }
+ return ""
+}
+
+func (x *RemoteAccessConfig) GetXdsPort() int64 {
+ if x != nil {
+ return x.XdsPort
+ }
+ return 0
+}
+
+func (x *RemoteAccessConfig) GetWebhookPort() int64 {
+ if x != nil {
+ return x.WebhookPort
+ }
+ return 0
+}
+
+func (x *RemoteAccessConfig) GetCertificateHosts() []string {
+ if x != nil {
+ return x.CertificateHosts
+ }
+ return nil
+}
+
+func (x *RemoteAccessConfig) GetGrpcPort() int64 {
+ if x != nil {
+ return x.GrpcPort
+ }
+ return 0
+}
+
+type EastWestGatewayEndpoint struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ ClusterName string `protobuf:"bytes,1,opt,name=clusterName,proto3" json:"clusterName,omitempty"`
+ Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"`
+ Port int64 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"`
+}
+
+func (x *EastWestGatewayEndpoint) Reset() {
+ *x = EastWestGatewayEndpoint{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_values_types_proto_msgTypes[4]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *EastWestGatewayEndpoint) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EastWestGatewayEndpoint) ProtoMessage() {}
+
+func (x *EastWestGatewayEndpoint) ProtoReflect() protoreflect.Message {
+ mi := &file_values_types_proto_msgTypes[4]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EastWestGatewayEndpoint.ProtoReflect.Descriptor instead.
+func (*EastWestGatewayEndpoint) Descriptor() ([]byte, []int) {
+ return file_values_types_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *EastWestGatewayEndpoint) GetClusterName() string {
+ if x != nil {
+ return x.ClusterName
+ }
+ return ""
+}
+
+func (x *EastWestGatewayEndpoint) GetAddress() string {
+ if x != nil {
+ return x.Address
+ }
+ return ""
+}
+
+func (x *EastWestGatewayEndpoint) GetPort() int64 {
+ if x != nil {
+ return x.Port
+ }
+ return 0
+}
+
+type EastWestGatewayConfig struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
+ ServiceType string `protobuf:"bytes,2,opt,name=serviceType,proto3" json:"serviceType,omitempty"`
+ Port int64 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"`
+ TargetPort int64 `protobuf:"varint,4,opt,name=targetPort,proto3" json:"targetPort,omitempty"`
+ NodePort int64 `protobuf:"varint,5,opt,name=nodePort,proto3" json:"nodePort,omitempty"`
+ Gateways []*EastWestGatewayEndpoint `protobuf:"bytes,6,rep,name=gateways,proto3" json:"gateways,omitempty"`
+ XdsAddress string `protobuf:"bytes,7,opt,name=xdsAddress,proto3" json:"xdsAddress,omitempty"`
+}
+
+func (x *EastWestGatewayConfig) Reset() {
+ *x = EastWestGatewayConfig{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_values_types_proto_msgTypes[5]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *EastWestGatewayConfig) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EastWestGatewayConfig) ProtoMessage() {}
+
+func (x *EastWestGatewayConfig) ProtoReflect() protoreflect.Message {
+ mi := &file_values_types_proto_msgTypes[5]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EastWestGatewayConfig.ProtoReflect.Descriptor instead.
+func (*EastWestGatewayConfig) Descriptor() ([]byte, []int) {
+ return file_values_types_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *EastWestGatewayConfig) GetEnabled() bool {
+ if x != nil {
+ return x.Enabled
+ }
+ return false
+}
+
+func (x *EastWestGatewayConfig) GetServiceType() string {
+ if x != nil {
+ return x.ServiceType
+ }
+ return ""
+}
+
+func (x *EastWestGatewayConfig) GetPort() int64 {
+ if x != nil {
+ return x.Port
+ }
+ return 0
+}
+
+func (x *EastWestGatewayConfig) GetTargetPort() int64 {
+ if x != nil {
+ return x.TargetPort
+ }
+ return 0
+}
+
+func (x *EastWestGatewayConfig) GetNodePort() int64 {
+ if x != nil {
+ return x.NodePort
+ }
+ return 0
+}
+
+func (x *EastWestGatewayConfig) GetGateways() []*EastWestGatewayEndpoint {
+ if x != nil {
+ return x.Gateways
+ }
+ return nil
+}
+
+func (x *EastWestGatewayConfig) GetXdsAddress() string {
+ if x != nil {
+ return x.XdsAddress
+ }
+ return ""
+}
+
+type MulticlusterConfig struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ RemoteAccess *RemoteAccessConfig `protobuf:"bytes,1,opt,name=remoteAccess,proto3" json:"remoteAccess,omitempty"`
+ EastWestGateway *EastWestGatewayConfig `protobuf:"bytes,2,opt,name=eastWestGateway,proto3" json:"eastWestGateway,omitempty"`
+}
+
+func (x *MulticlusterConfig) Reset() {
+ *x = MulticlusterConfig{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_values_types_proto_msgTypes[6]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *MulticlusterConfig) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MulticlusterConfig) ProtoMessage() {}
+
+func (x *MulticlusterConfig) ProtoReflect() protoreflect.Message {
+ mi := &file_values_types_proto_msgTypes[6]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use MulticlusterConfig.ProtoReflect.Descriptor instead.
+func (*MulticlusterConfig) Descriptor() ([]byte, []int) {
+ return file_values_types_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *MulticlusterConfig) GetRemoteAccess() *RemoteAccessConfig {
+ if x != nil {
+ return x.RemoteAccess
+ }
+ return nil
+}
+
+func (x *MulticlusterConfig) GetEastWestGateway() *EastWestGatewayConfig {
+ if x != nil {
+ return x.EastWestGateway
+ }
+ return nil
+}
+
type Values struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -224,7 +532,7 @@ type Values struct {
func (x *Values) Reset() {
*x = Values{}
if protoimpl.UnsafeEnabled {
- mi := &file_values_types_proto_msgTypes[3]
+ mi := &file_values_types_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -237,7 +545,7 @@ func (x *Values) String() string {
func (*Values) ProtoMessage() {}
func (x *Values) ProtoReflect() protoreflect.Message {
- mi := &file_values_types_proto_msgTypes[3]
+ mi := &file_values_types_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -250,7 +558,7 @@ func (x *Values) ProtoReflect() protoreflect.Message {
// Deprecated: Use Values.ProtoReflect.Descriptor instead.
func (*Values) Descriptor() ([]byte, []int) {
- return file_values_types_proto_rawDescGZIP(), []int{3}
+ return file_values_types_proto_rawDescGZIP(), []int{7}
}
func (x *Values) GetGlobal() *GlobalConfig {
@@ -289,7 +597,7 @@ type IntOrString struct {
func (x *IntOrString) Reset() {
*x = IntOrString{}
if protoimpl.UnsafeEnabled {
- mi := &file_values_types_proto_msgTypes[4]
+ mi := &file_values_types_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -302,7 +610,7 @@ func (x *IntOrString) String() string {
func (*IntOrString) ProtoMessage() {}
func (x *IntOrString) ProtoReflect() protoreflect.Message {
- mi := &file_values_types_proto_msgTypes[4]
+ mi := &file_values_types_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -315,7 +623,7 @@ func (x *IntOrString) ProtoReflect() protoreflect.Message {
// Deprecated: Use IntOrString.ProtoReflect.Descriptor instead.
func (*IntOrString) Descriptor() ([]byte, []int) {
- return file_values_types_proto_rawDescGZIP(), []int{4}
+ return file_values_types_proto_rawDescGZIP(), []int{8}
}
func (x *IntOrString) GetType() int64 {
@@ -354,7 +662,7 @@ var file_values_types_proto_rawDesc = []byte{
0x6f, 0x78, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x63, 0x6c, 0x75,
0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22,
- 0xcc, 0x01, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+ 0x9d, 0x02, 0x0a, 0x0c, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x12, 0x3a, 0x0a, 0x05, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x24, 0x2e, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43,
@@ -366,26 +674,81 @@ var file_values_types_proto_rawDesc = []byte{
0x68, 0x61, 0x31, 0x2e, 0x47, 0x55, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x67,
0x75, 0x69, 0x12, 0x2a, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, 0x69,
0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x63, 0x6f,
- 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x63,
- 0x0a, 0x06, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62,
- 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x64, 0x75, 0x62, 0x62, 0x6f,
- 0x2e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
- 0x61, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
- 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73,
- 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73,
- 0x69, 0x6f, 0x6e, 0x22, 0x8c, 0x01, 0x0a, 0x0b, 0x49, 0x6e, 0x74, 0x4f, 0x72, 0x53, 0x74, 0x72,
- 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
- 0x03, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x69, 0x6e, 0x74, 0x56, 0x61,
- 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
- 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56,
- 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x34, 0x0a, 0x06,
- 0x73, 0x74, 0x72, 0x56, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67,
- 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53,
- 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x73, 0x74, 0x72, 0x56,
- 0x61, 0x6c, 0x42, 0x2f, 0x5a, 0x2d, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x61, 0x70, 0x61, 0x63,
- 0x68, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2f, 0x64, 0x75, 0x62,
- 0x62, 0x6f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61,
- 0x70, 0x69, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f,
+ 0x0a, 0x0c, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x05,
+ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x6f, 0x70, 0x65,
+ 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d,
+ 0x75, 0x6c, 0x74, 0x69, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69,
+ 0x67, 0x52, 0x0c, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x22,
+ 0xd4, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73,
+ 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65,
+ 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64,
+ 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18,
+ 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79,
+ 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x78, 0x64, 0x73, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20,
+ 0x01, 0x28, 0x03, 0x52, 0x07, 0x78, 0x64, 0x73, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b,
+ 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28,
+ 0x03, 0x52, 0x0b, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2a,
+ 0x0a, 0x10, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x73,
+ 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66,
+ 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x67, 0x72,
+ 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x67, 0x72,
+ 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x69, 0x0a, 0x17, 0x45, 0x61, 0x73, 0x74, 0x57, 0x65,
+ 0x73, 0x74, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+ 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65,
+ 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e,
+ 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02,
+ 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a,
+ 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x70, 0x6f, 0x72,
+ 0x74, 0x22, 0x91, 0x02, 0x0a, 0x15, 0x45, 0x61, 0x73, 0x74, 0x57, 0x65, 0x73, 0x74, 0x47, 0x61,
+ 0x74, 0x65, 0x77, 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x65,
+ 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e,
+ 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+ 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76,
+ 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18,
+ 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x74,
+ 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52,
+ 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6e,
+ 0x6f, 0x64, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6e,
+ 0x6f, 0x64, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x4c, 0x0a, 0x08, 0x67, 0x61, 0x74, 0x65, 0x77,
+ 0x61, 0x79, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x75, 0x62, 0x62,
+ 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70,
+ 0x68, 0x61, 0x31, 0x2e, 0x45, 0x61, 0x73, 0x74, 0x57, 0x65, 0x73, 0x74, 0x47, 0x61, 0x74, 0x65,
+ 0x77, 0x61, 0x79, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x08, 0x67, 0x61, 0x74,
+ 0x65, 0x77, 0x61, 0x79, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x78, 0x64, 0x73, 0x41, 0x64, 0x64, 0x72,
+ 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x78, 0x64, 0x73, 0x41, 0x64,
+ 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xbf, 0x01, 0x0a, 0x12, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x63,
+ 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4f, 0x0a, 0x0c,
+ 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01,
+ 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x72, 0x61,
+ 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x6d,
+ 0x6f, 0x74, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
+ 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x58, 0x0a,
+ 0x0f, 0x65, 0x61, 0x73, 0x74, 0x57, 0x65, 0x73, 0x74, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79,
+ 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x6f,
+ 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
+ 0x2e, 0x45, 0x61, 0x73, 0x74, 0x57, 0x65, 0x73, 0x74, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79,
+ 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x65, 0x61, 0x73, 0x74, 0x57, 0x65, 0x73, 0x74,
+ 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x22, 0x63, 0x0a, 0x06, 0x56, 0x61, 0x6c, 0x75, 0x65,
+ 0x73, 0x12, 0x3d, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28,
+ 0x0b, 0x32, 0x25, 0x2e, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74,
+ 0x6f, 0x72, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x6c, 0x6f, 0x62,
+ 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c,
+ 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x8c, 0x01, 0x0a,
+ 0x0b, 0x49, 0x6e, 0x74, 0x4f, 0x72, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04,
+ 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65,
+ 0x12, 0x33, 0x0a, 0x06, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
+ 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
+ 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x69,
+ 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x34, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x56, 0x61, 0x6c, 0x18,
+ 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
+ 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61,
+ 0x6c, 0x75, 0x65, 0x52, 0x06, 0x73, 0x74, 0x72, 0x56, 0x61, 0x6c, 0x42, 0x2f, 0x5a, 0x2d, 0x64,
+ 0x75, 0x62, 0x62, 0x6f, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x2f,
+ 0x64, 0x75, 0x62, 0x62, 0x6f, 0x2f, 0x64, 0x75, 0x62, 0x62, 0x6f, 0x6f, 0x70, 0x65, 0x72, 0x61,
+ 0x74, 0x6f, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x62, 0x06, 0x70, 0x72,
+ 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -400,27 +763,35 @@ func file_values_types_proto_rawDescGZIP() []byte {
return file_values_types_proto_rawDescData
}
-var file_values_types_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_values_types_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
var file_values_types_proto_goTypes = []interface{}{
- (*GUIConfig)(nil), // 0: dubbo.operator.v1alpha1.GUIConfig
- (*ProxyConfig)(nil), // 1: dubbo.operator.v1alpha1.ProxyConfig
- (*GlobalConfig)(nil), // 2: dubbo.operator.v1alpha1.GlobalConfig
- (*Values)(nil), // 3: dubbo.operator.v1alpha1.Values
- (*IntOrString)(nil), // 4: dubbo.operator.v1alpha1.IntOrString
- (*wrapperspb.Int32Value)(nil), // 5: google.protobuf.Int32Value
- (*wrapperspb.StringValue)(nil), // 6: google.protobuf.StringValue
+ (*GUIConfig)(nil), // 0: dubbo.operator.v1alpha1.GUIConfig
+ (*ProxyConfig)(nil), // 1: dubbo.operator.v1alpha1.ProxyConfig
+ (*GlobalConfig)(nil), // 2: dubbo.operator.v1alpha1.GlobalConfig
+ (*RemoteAccessConfig)(nil), // 3: dubbo.operator.v1alpha1.RemoteAccessConfig
+ (*EastWestGatewayEndpoint)(nil), // 4: dubbo.operator.v1alpha1.EastWestGatewayEndpoint
+ (*EastWestGatewayConfig)(nil), // 5: dubbo.operator.v1alpha1.EastWestGatewayConfig
+ (*MulticlusterConfig)(nil), // 6: dubbo.operator.v1alpha1.MulticlusterConfig
+ (*Values)(nil), // 7: dubbo.operator.v1alpha1.Values
+ (*IntOrString)(nil), // 8: dubbo.operator.v1alpha1.IntOrString
+ (*wrapperspb.Int32Value)(nil), // 9: google.protobuf.Int32Value
+ (*wrapperspb.StringValue)(nil), // 10: google.protobuf.StringValue
}
var file_values_types_proto_depIdxs = []int32{
- 1, // 0: dubbo.operator.v1alpha1.GlobalConfig.proxy:type_name -> dubbo.operator.v1alpha1.ProxyConfig
- 0, // 1: dubbo.operator.v1alpha1.GlobalConfig.gui:type_name -> dubbo.operator.v1alpha1.GUIConfig
- 2, // 2: dubbo.operator.v1alpha1.Values.global:type_name -> dubbo.operator.v1alpha1.GlobalConfig
- 5, // 3: dubbo.operator.v1alpha1.IntOrString.intVal:type_name -> google.protobuf.Int32Value
- 6, // 4: dubbo.operator.v1alpha1.IntOrString.strVal:type_name -> google.protobuf.StringValue
- 5, // [5:5] is the sub-list for method output_type
- 5, // [5:5] is the sub-list for method input_type
- 5, // [5:5] is the sub-list for extension type_name
- 5, // [5:5] is the sub-list for extension extendee
- 0, // [0:5] is the sub-list for field type_name
+ 1, // 0: dubbo.operator.v1alpha1.GlobalConfig.proxy:type_name -> dubbo.operator.v1alpha1.ProxyConfig
+ 0, // 1: dubbo.operator.v1alpha1.GlobalConfig.gui:type_name -> dubbo.operator.v1alpha1.GUIConfig
+ 6, // 2: dubbo.operator.v1alpha1.GlobalConfig.multicluster:type_name -> dubbo.operator.v1alpha1.MulticlusterConfig
+ 4, // 3: dubbo.operator.v1alpha1.EastWestGatewayConfig.gateways:type_name -> dubbo.operator.v1alpha1.EastWestGatewayEndpoint
+ 3, // 4: dubbo.operator.v1alpha1.MulticlusterConfig.remoteAccess:type_name -> dubbo.operator.v1alpha1.RemoteAccessConfig
+ 5, // 5: dubbo.operator.v1alpha1.MulticlusterConfig.eastWestGateway:type_name -> dubbo.operator.v1alpha1.EastWestGatewayConfig
+ 2, // 6: dubbo.operator.v1alpha1.Values.global:type_name -> dubbo.operator.v1alpha1.GlobalConfig
+ 9, // 7: dubbo.operator.v1alpha1.IntOrString.intVal:type_name -> google.protobuf.Int32Value
+ 10, // 8: dubbo.operator.v1alpha1.IntOrString.strVal:type_name -> google.protobuf.StringValue
+ 9, // [9:9] is the sub-list for method output_type
+ 9, // [9:9] is the sub-list for method input_type
+ 9, // [9:9] is the sub-list for extension type_name
+ 9, // [9:9] is the sub-list for extension extendee
+ 0, // [0:9] is the sub-list for field type_name
}
func init() { file_values_types_proto_init() }
@@ -466,7 +837,7 @@ func file_values_types_proto_init() {
}
}
file_values_types_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*Values); i {
+ switch v := v.(*RemoteAccessConfig); i {
case 0:
return &v.state
case 1:
@@ -478,6 +849,54 @@ func file_values_types_proto_init() {
}
}
file_values_types_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*EastWestGatewayEndpoint); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_values_types_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*EastWestGatewayConfig); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_values_types_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*MulticlusterConfig); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_values_types_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*Values); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_values_types_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IntOrString); i {
case 0:
return &v.state
@@ -496,7 +915,7 @@ func file_values_types_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_values_types_proto_rawDesc,
NumEnums: 0,
- NumMessages: 5,
+ NumMessages: 9,
NumExtensions: 0,
NumServices: 0,
},
diff --git a/dubbooperator/pkg/render/manifest_test.go b/dubbooperator/pkg/render/manifest_test.go
index 940edec6..313b19c8 100644
--- a/dubbooperator/pkg/render/manifest_test.go
+++ b/dubbooperator/pkg/render/manifest_test.go
@@ -76,6 +76,121 @@ func TestGenerateManifestExposesDubbodPrometheusScrapeEndpoint(t *testing.T) {
t.Fatal("dubbod Service not rendered")
}
+func TestGenerateManifestRemoteAccessServiceIsOptional(t *testing.T) {
+ manifests, _, err := GenerateManifest(nil, nil, nil, nil)
+ if err != nil {
+ t.Fatalf("GenerateManifest() error = %v", err)
+ }
+ if hasManifest(manifests, "Service", "dubbod-remote") {
+ t.Fatal("dubbod-remote rendered while remote access is disabled")
+ }
+
+ manifests, _, err = GenerateManifest(nil, []string{
+ "values.global.multicluster.remoteAccess.enabled=true",
+ "values.global.multicluster.remoteAccess.serviceType=NodePort",
+ "values.global.multicluster.remoteAccess.grpcPort=32010",
+ "values.global.multicluster.remoteAccess.xdsPort=26112",
+ "values.global.multicluster.remoteAccess.webhookPort=30443",
+ }, nil, nil)
+ if err != nil {
+ t.Fatalf("GenerateManifest() with remote access error = %v", err)
+ }
+ service := findManifest(t, manifests, "Service", "dubbod-remote")
+ serviceType, _, _ := unstructured.NestedString(service.Object, "spec", "type")
+ if serviceType != "NodePort" {
+ t.Fatalf("dubbod-remote service type = %q, want NodePort", serviceType)
+ }
+ grpcPort, _, _ := unstructured.NestedInt64(findPort(t, service, "grpc-xds"), "port")
+ xdsPort, _, _ := unstructured.NestedInt64(findPort(t, service, "tls-xds"), "port")
+ webhookPort, _, _ := unstructured.NestedInt64(findPort(t, service, "https-webhook"), "port")
+ if grpcPort != 32010 || xdsPort != 26112 || webhookPort != 30443 {
+ t.Fatalf("dubbod-remote ports = %d/%d/%d, want 32010/26112/30443", grpcPort, xdsPort, webhookPort)
+ }
+}
+
+func TestGenerateManifestRemoteAccessCertificateHosts(t *testing.T) {
+ manifests, _, err := GenerateManifest(nil, []string{
+ "values.global.multicluster.remoteAccess.enabled=true",
+ "values.global.multicluster.remoteAccess.certificateHosts[0]=192.168.15.164",
+ "values.global.multicluster.remoteAccess.certificateHosts[1]=dubbod.example.internal",
+ }, nil, nil)
+ if err != nil {
+ t.Fatalf("GenerateManifest() with remote access certificate hosts error = %v", err)
+ }
+
+ deployment := findManifest(t, manifests, "Deployment", "dubbod")
+ containers, ok, err := unstructured.NestedSlice(deployment.Object, "spec", "template", "spec", "containers")
+ if err != nil || !ok || len(containers) == 0 {
+ t.Fatalf("deployment containers missing: ok=%v err=%v", ok, err)
+ }
+ execute := containers[0].(map[string]interface{})
+ env, ok, err := unstructured.NestedSlice(execute, "env")
+ if err != nil || !ok {
+ t.Fatalf("deployment env missing: ok=%v err=%v", ok, err)
+ }
+ for _, item := range env {
+ entry, ok := item.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ name, _, _ := unstructured.NestedString(entry, "name")
+ if name != "DUBBOD_CUSTOM_HOST" {
+ continue
+ }
+ value, _, _ := unstructured.NestedString(entry, "value")
+ if value != "192.168.15.164,dubbod.example.internal" {
+ t.Fatalf("DUBBOD_CUSTOM_HOST = %q", value)
+ }
+ return
+ }
+ t.Fatal("dubbod deployment missing DUBBOD_CUSTOM_HOST")
+}
+
+func TestGenerateManifestEastWestGatewayConfig(t *testing.T) {
+ manifests, _, err := GenerateManifest(nil, []string{
+ "values.global.multicluster.eastWestGateway.enabled=true",
+ "values.global.multicluster.eastWestGateway.serviceType=NodePort",
+ "values.global.multicluster.eastWestGateway.port=15443",
+ "values.global.multicluster.eastWestGateway.targetPort=15080",
+ "values.global.multicluster.eastWestGateway.nodePort=32443",
+ "values.global.multicluster.eastWestGateway.xdsAddress=http://192.168.15.164:32010",
+ "values.global.multicluster.eastWestGateway.gateways[0].clusterName=remote",
+ "values.global.multicluster.eastWestGateway.gateways[0].address=192.168.15.155",
+ "values.global.multicluster.eastWestGateway.gateways[0].port=15443",
+ }, nil, nil)
+ if err != nil {
+ t.Fatalf("GenerateManifest() with east-west gateway error = %v", err)
+ }
+
+ gateway := findManifest(t, manifests, "Gateway", "dubbod-eastwest-gateway")
+ if got := gateway.GetAnnotations()["gateway.dubbo.apache.org/service-type"]; got != "NodePort" {
+ t.Fatalf("east-west gateway service-type annotation = %q, want NodePort", got)
+ }
+ if got := gateway.GetAnnotations()["gateway.dubbo.apache.org/target-port"]; got != "15080" {
+ t.Fatalf("east-west gateway target-port annotation = %q, want 15080", got)
+ }
+ if got := gateway.GetAnnotations()["gateway.dubbo.apache.org/node-port"]; got != "32443" {
+ t.Fatalf("east-west gateway node-port annotation = %q, want 32443", got)
+ }
+ if got := gateway.GetAnnotations()["gateway.dubbo.apache.org/xds-address"]; got != "http://192.168.15.164:32010" {
+ t.Fatalf("east-west gateway xds-address annotation = %q, want override", got)
+ }
+ listeners, ok, err := unstructured.NestedSlice(gateway.Object, "spec", "listeners")
+ if err != nil || !ok || len(listeners) != 1 {
+ t.Fatalf("gateway listeners missing: ok=%v err=%v", ok, err)
+ }
+ listener := listeners[0].(map[string]interface{})
+ port, _, _ := unstructured.NestedInt64(listener, "port")
+ if port != 15443 {
+ t.Fatalf("east-west listener port = %d, want 15443", port)
+ }
+
+ deployment := findManifest(t, manifests, "Deployment", "dubbod")
+ if got := deploymentEnvValue(t, deployment, "DUBBO_EASTWEST_GATEWAYS"); got != "remote=192.168.15.155:15443" {
+ t.Fatalf("DUBBO_EASTWEST_GATEWAYS = %q", got)
+ }
+}
+
func TestGenerateManifestSetOverridesHelmValues(t *testing.T) {
manifests, _, err := GenerateManifest(nil, []string{
"values.global.gui.port=26081",
@@ -179,6 +294,33 @@ func TestGenerateManifestRejectsRemovedInstallSurface(t *testing.T) {
}
}
+func deploymentEnvValue(t *testing.T, deployment manifest.Manifest, name string) string {
+ t.Helper()
+ containers, ok, err := unstructured.NestedSlice(deployment.Object, "spec", "template", "spec", "containers")
+ if err != nil || !ok || len(containers) == 0 {
+ t.Fatalf("deployment containers missing: ok=%v err=%v", ok, err)
+ }
+ execute := containers[0].(map[string]interface{})
+ env, ok, err := unstructured.NestedSlice(execute, "env")
+ if err != nil || !ok {
+ t.Fatalf("deployment env missing: ok=%v err=%v", ok, err)
+ }
+ for _, item := range env {
+ entry, ok := item.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ gotName, _, _ := unstructured.NestedString(entry, "name")
+ if gotName != name {
+ continue
+ }
+ value, _, _ := unstructured.NestedString(entry, "value")
+ return value
+ }
+ t.Fatalf("deployment missing env %s", name)
+ return ""
+}
+
func findManifest(t *testing.T, manifests []manifest.ManifestSet, kind, name string) manifest.Manifest {
t.Helper()
if m, ok := findOptionalManifest(manifests, kind, name); ok {
diff --git a/pkg/kube/inject/proxyless_test.go b/pkg/kube/inject/proxyless_test.go
index 8cc9fd76..4fb61bc7 100644
--- a/pkg/kube/inject/proxyless_test.go
+++ b/pkg/kube/inject/proxyless_test.go
@@ -318,6 +318,55 @@ func TestAddApplicationContainerConfigInjectsProxylessGRPCContract(t *testing.T)
}
}
+func TestAddApplicationContainerConfigOverridesRemoteClusterEnvs(t *testing.T) {
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "grpc-provider",
+ Namespace: "grpc-app",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{{
+ Name: "app",
+ }},
+ },
+ }
+ req := InjectionParameters{
+ meshConfig: &meshv1alpha1.MeshConfig{
+ DefaultConfig: &meshv1alpha1.ProxyConfig{
+ DiscoveryAddress: "dubbod.dubbo-system.svc:26012",
+ },
+ },
+ proxyEnvs: map[string]string{
+ ProxylessXDSAddressEnvName: "192.168.15.164:32049",
+ "CA_ADDRESS": "192.168.15.164:32049",
+ "DUBBO_META_CLUSTER_ID": "remote",
+ },
+ }
+ if err := addApplicationContainerConfig(pod, req); err != nil {
+ t.Fatalf("addApplicationContainerConfig() failed: %v", err)
+ }
+ container := pod.Spec.Containers[0]
+ for name, want := range req.proxyEnvs {
+ if !hasEnv(container.Env, name, want) {
+ t.Fatalf("%s env missing override %q", name, want)
+ }
+ }
+}
+
+func TestParseInjectEnvsForRemoteClusterPath(t *testing.T) {
+ got := parseInjectEnvs("/inject/DUBBO_META_CLUSTER_ID/remote/XDS_ADDRESS/192.168.15.164:32049/CA_ADDRESS/192.168.15.164:32049")
+ want := map[string]string{
+ "DUBBO_META_CLUSTER_ID": "remote",
+ "XDS_ADDRESS": "192.168.15.164:32049",
+ "CA_ADDRESS": "192.168.15.164:32049",
+ }
+ for name, value := range want {
+ if got[name] != value {
+ t.Fatalf("%s = %q, want %q", name, got[name], value)
+ }
+ }
+}
+
func TestInstallerGRPCEngineTemplateConfiguresXDSClientForDubbodImage(t *testing.T) {
_, currentFile, _, ok := runtime.Caller(0)
if !ok {
@@ -368,6 +417,11 @@ func TestInstallerGRPCEngineTemplateConfiguresXDSClientForDubbodImage(t *testing
proxyConfig: &meshv1alpha1.ProxyConfig{
DiscoveryAddress: "dubbod.dubbo-system.svc:26012",
},
+ proxyEnvs: map[string]string{
+ ProxylessXDSAddressEnvName: "192.168.15.164:32049",
+ "CA_ADDRESS": "192.168.15.164:32049",
+ "DUBBO_META_CLUSTER_ID": "remote",
+ },
}
mergedPod, injectedPod, err := RunTemplate(req)
@@ -383,6 +437,11 @@ func TestInstallerGRPCEngineTemplateConfiguresXDSClientForDubbodImage(t *testing
if strings.Join(container.Args, ",") != strings.Join(wantArgs, ",") {
t.Fatalf("args = %v, want %v", container.Args, wantArgs)
}
+ for name, want := range req.proxyEnvs {
+ if !hasEnv(container.Env, name, want) {
+ t.Fatalf("%s env missing override %q", name, want)
+ }
+ }
}
func TestInstallerGRPCEngineTemplateDoesNotConfigureXDSClientForNonDubbodImage(t *testing.T) {
diff --git a/pkg/kube/inject/webhook.go b/pkg/kube/inject/webhook.go
index 760b932b..cdf183dc 100644
--- a/pkg/kube/inject/webhook.go
+++ b/pkg/kube/inject/webhook.go
@@ -614,6 +614,9 @@ func addApplicationContainerConfig(pod *corev1.Pod, req InjectionParameters) err
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.hostIP"},
},
})
+ if len(req.proxyEnvs) > 0 {
+ updateClusterEnvs(container, req.proxyEnvs)
+ }
hasProxyVolumeMount := false
for _, vm := range container.VolumeMounts {
@@ -819,6 +822,7 @@ func parseInjectEnvs(path string) map[string]string {
webhookLog.Warnf("Add number of inject env entries, ignore the last key %s\n", k)
break
}
+ newEnvs[k] = res[i+1]
}
return newEnvs
diff --git a/pkg/kube/multicluster/cluster.go b/pkg/kube/multicluster/cluster.go
index 50e489f9..bba2e3a6 100644
--- a/pkg/kube/multicluster/cluster.go
+++ b/pkg/kube/multicluster/cluster.go
@@ -18,6 +18,8 @@ package multicluster
import (
"crypto/sha256"
+ "sync"
+ "time"
"github.com/apache/dubbo-kubernetes/pkg/cluster"
"github.com/apache/dubbo-kubernetes/pkg/kube"
@@ -34,6 +36,7 @@ type Cluster struct {
initialSync *atomic.Bool
initialSyncTimeout *atomic.Bool
+ closeOnce sync.Once
}
func (c *Cluster) HasSynced() bool {
@@ -46,6 +49,41 @@ func (c *Cluster) HasSynced() bool {
return c.initialSync.Load() || c.initialSyncTimeout.Load()
}
+func NewRemoteCluster(id cluster.ID, client kube.Client, kubeConfigSha [sha256.Size]byte) *Cluster {
+ return &Cluster{
+ ID: id,
+ Client: client,
+ kubeConfigSha: kubeConfigSha,
+ stop: make(chan struct{}),
+ initialSync: atomic.NewBool(false),
+ initialSyncTimeout: atomic.NewBool(false),
+ }
+}
+
+func (c *Cluster) Start(syncTimeout time.Duration) {
+ if syncTimeout > 0 {
+ time.AfterFunc(syncTimeout, func() {
+ if !c.initialSync.Load() && !c.Closed() {
+ c.initialSyncTimeout.Store(true)
+ }
+ })
+ }
+ go func() {
+ if c.Client.RunAndWait(c.stop) {
+ c.initialSync.Store(true)
+ }
+ }()
+}
+
+func (c *Cluster) Close() {
+ c.closeOnce.Do(func() {
+ close(c.stop)
+ if c.Client != nil {
+ c.Client.Shutdown()
+ }
+ })
+}
+
func (c *Cluster) Closed() bool {
select {
case <-c.stop:
diff --git a/pkg/kube/multicluster/clusterstore.go b/pkg/kube/multicluster/clusterstore.go
index 1a19f25d..4da7245b 100644
--- a/pkg/kube/multicluster/clusterstore.go
+++ b/pkg/kube/multicluster/clusterstore.go
@@ -19,16 +19,61 @@ package multicluster
import (
"sync"
+ "github.com/apache/dubbo-kubernetes/pkg/cluster"
"github.com/apache/dubbo-kubernetes/pkg/util/sets"
)
type ClusterStore struct {
sync.RWMutex
- clusters sets.String
+ clusters map[cluster.ID]*Cluster
+}
+
+func NewClusterStore() *ClusterStore {
+ return &ClusterStore{
+ clusters: make(map[cluster.ID]*Cluster),
+ }
+}
+
+func (c *ClusterStore) Get(id cluster.ID) (*Cluster, bool) {
+ c.RLock()
+ defer c.RUnlock()
+ cluster, ok := c.clusters[id]
+ return cluster, ok
+}
+
+func (c *ClusterStore) Store(cluster *Cluster) {
+ c.Lock()
+ defer c.Unlock()
+ c.clusters[cluster.ID] = cluster
+}
+
+func (c *ClusterStore) Delete(id cluster.ID) (*Cluster, bool) {
+ c.Lock()
+ defer c.Unlock()
+ cluster, ok := c.clusters[id]
+ if ok {
+ delete(c.clusters, id)
+ }
+ return cluster, ok
+}
+
+func (c *ClusterStore) IDs() sets.String {
+ c.RLock()
+ defer c.RUnlock()
+ out := sets.New[string]()
+ for id := range c.clusters {
+ out.Insert(string(id))
+ }
+ return out
}
func (c *ClusterStore) HasSynced() bool {
c.RLock()
defer c.RUnlock()
+ for _, cluster := range c.clusters {
+ if !cluster.HasSynced() {
+ return false
+ }
+ }
return true
}
diff --git a/pkg/kube/multicluster/eastwest.go b/pkg/kube/multicluster/eastwest.go
new file mode 100644
index 00000000..247f453b
--- /dev/null
+++ b/pkg/kube/multicluster/eastwest.go
@@ -0,0 +1,55 @@
+package multicluster
+
+import (
+ "fmt"
+ "net"
+ "strconv"
+ "strings"
+
+ "github.com/apache/dubbo-kubernetes/pkg/cluster"
+)
+
+const EastWestGatewayEnvName = "DUBBO_EASTWEST_GATEWAYS"
+
+type EastWestGateway struct {
+ Cluster cluster.ID
+ Address string
+ Port uint32
+}
+
+func (g EastWestGateway) Endpoint() string {
+ return net.JoinHostPort(g.Address, strconv.Itoa(int(g.Port)))
+}
+
+func ParseEastWestGateways(raw string) (map[cluster.ID]EastWestGateway, error) {
+ raw = strings.TrimSpace(raw)
+ if raw == "" {
+ return nil, nil
+ }
+ out := map[cluster.ID]EastWestGateway{}
+ for _, item := range strings.Split(raw, ",") {
+ item = strings.TrimSpace(item)
+ if item == "" {
+ continue
+ }
+ parts := strings.SplitN(item, "=", 2)
+ if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" {
+ return nil, fmt.Errorf("invalid east-west gateway entry %q, expected cluster=host:port", item)
+ }
+ clusterID := cluster.ID(strings.TrimSpace(parts[0]))
+ host, portText, err := net.SplitHostPort(strings.TrimSpace(parts[1]))
+ if err != nil {
+ return nil, fmt.Errorf("invalid east-west gateway endpoint %q: %w", parts[1], err)
+ }
+ port, err := strconv.ParseUint(portText, 10, 32)
+ if err != nil || port == 0 || port > 65535 {
+ return nil, fmt.Errorf("invalid east-west gateway port %q", portText)
+ }
+ out[clusterID] = EastWestGateway{
+ Cluster: clusterID,
+ Address: host,
+ Port: uint32(port),
+ }
+ }
+ return out, nil
+}
diff --git a/pkg/kube/multicluster/eastwest_test.go b/pkg/kube/multicluster/eastwest_test.go
new file mode 100644
index 00000000..3e03a15f
--- /dev/null
+++ b/pkg/kube/multicluster/eastwest_test.go
@@ -0,0 +1,35 @@
+package multicluster
+
+import (
+ "testing"
+
+ "github.com/apache/dubbo-kubernetes/pkg/cluster"
+)
+
+func TestParseEastWestGateways(t *testing.T) {
+ got, err := ParseEastWestGateways("remote=192.168.15.155:15443,primary=dubbod-eastwest.example.com:443")
+ if err != nil {
+ t.Fatalf("ParseEastWestGateways() error = %v", err)
+ }
+ if got[cluster.ID("remote")].Address != "192.168.15.155" || got[cluster.ID("remote")].Port != 15443 {
+ t.Fatalf("remote gateway = %#v", got[cluster.ID("remote")])
+ }
+ if got[cluster.ID("primary")].Endpoint() != "dubbod-eastwest.example.com:443" {
+ t.Fatalf("primary endpoint = %q", got[cluster.ID("primary")].Endpoint())
+ }
+}
+
+func TestParseEastWestGatewaysRejectsBadEntries(t *testing.T) {
+ for _, raw := range []string{
+ "remote",
+ "remote=192.168.15.155",
+ "remote=192.168.15.155:notaport",
+ "remote=192.168.15.155:0",
+ } {
+ t.Run(raw, func(t *testing.T) {
+ if _, err := ParseEastWestGateways(raw); err == nil {
+ t.Fatal("ParseEastWestGateways() returned nil error")
+ }
+ })
+ }
+}
diff --git a/pkg/kube/multicluster/secretcontroller.go b/pkg/kube/multicluster/secretcontroller.go
index 5ca1a9d0..f3068f6b 100644
--- a/pkg/kube/multicluster/secretcontroller.go
+++ b/pkg/kube/multicluster/secretcontroller.go
@@ -17,15 +17,35 @@
package multicluster
import (
+ "crypto/sha256"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+
"github.com/apache/dubbo-kubernetes/pkg/cluster"
"github.com/apache/dubbo-kubernetes/pkg/config/mesh"
"github.com/apache/dubbo-kubernetes/pkg/kube"
"github.com/apache/dubbo-kubernetes/pkg/kube/controllers"
"github.com/apache/dubbo-kubernetes/pkg/kube/kclient"
+ dubbolog "github.com/apache/dubbo-kubernetes/pkg/log"
corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+ RemoteClusterSecretLabel = "dubbo.apache.org/remote-cluster"
+ RemoteClusterSecretLabelValue = "true"
+ RemoteClusterSecretKubeconfigKey = "kubeconfig"
+ RemoteClusterSecretClusterNameKey = "cluster-name"
+
+ defaultRemoteClusterSyncTimeout = 30 * time.Second
)
+var log = dubbolog.RegisterScope("multicluster", "multicluster debugging")
+
type handler interface {
clusterAdded(cluster *Cluster) ComponentConstraint
clusterUpdated(cluster *Cluster) ComponentConstraint
@@ -42,6 +62,7 @@ type Controller struct {
configClusterID cluster.ID
configCluster *Cluster
configClusterSyncers []ComponentConstraint
+ syncTimeout time.Duration
queue controllers.Queue
secrets kclient.Client[*corev1.Secret]
@@ -51,6 +72,12 @@ type Controller struct {
meshWatcher mesh.Watcher
handlers []handler
+
+ mu sync.RWMutex
+ secretToCluster map[types.NamespacedName]cluster.ID
+ clusterToSecret map[cluster.ID]types.NamespacedName
+ newRemoteClient func([]byte, cluster.ID, ...func(*rest.Config)) (kube.Client, error)
+ startRemoteCluster func(*Cluster)
}
func NewController(kubeclientset kube.Client, namespace string, clusterID cluster.ID,
@@ -60,10 +87,26 @@ func NewController(kubeclientset kube.Client, namespace string, clusterID cluste
namespace: namespace,
configClusterID: clusterID,
configCluster: &Cluster{Client: kubeclientset, ID: clusterID},
-
+ syncTimeout: defaultRemoteClusterSyncTimeout,
configOverrides: configOverrides,
meshWatcher: meshWatcher,
+ cs: NewClusterStore(),
+ secretToCluster: make(map[types.NamespacedName]cluster.ID),
+ clusterToSecret: make(map[cluster.ID]types.NamespacedName),
+ newRemoteClient: newRemoteClient,
+ }
+ controller.startRemoteCluster = func(cluster *Cluster) {
+ cluster.Start(controller.syncTimeout)
}
+ controller.queue = controllers.NewQueue("multicluster secret controller",
+ controllers.WithReconciler(controller.reconcile),
+ controllers.WithMaxAttempts(5),
+ )
+ controller.secrets = kclient.NewFiltered[*corev1.Secret](kubeclientset, kclient.Filter{
+ Namespace: namespace,
+ LabelSelector: RemoteClusterSecretLabel + "=" + RemoteClusterSecretLabelValue,
+ })
+ controller.secrets.AddEventHandler(controllers.ObjectHandler(controller.queue.AddObject))
return controller
}
@@ -80,6 +123,14 @@ func (c *Controller) Run(stopCh <-chan struct{}) error {
// run handlers for the config cluster; do not store this *Cluster in the ClusterStore or give it a SyncTimeout
// this is done outside the goroutine, we should block other Run/startFuncs until this is registered
c.configClusterSyncers = c.handleAdd(c.configCluster)
+ go func() {
+ c.secrets.Start(stopCh)
+ if !kube.WaitForCacheSync("multicluster secrets", stopCh, c.secrets.HasSynced) {
+ c.queue.ShutDownEarly()
+ return
+ }
+ c.queue.Run(stopCh)
+ }()
return nil
}
@@ -114,3 +165,119 @@ func (c *Controller) registerHandler(h handler) {
// Intentionally no lock. The controller today requires that handlers are registered before execution and not in parallel.
c.handlers = append(c.handlers, h)
}
+
+func (c *Controller) reconcile(key types.NamespacedName) error {
+ secret := c.secrets.Get(key.Name, key.Namespace)
+ if secret == nil {
+ c.deleteSecret(key)
+ return nil
+ }
+ if err := c.reconcileSecret(key, secret); err != nil {
+ log.Warnf("ignoring remote cluster secret %s/%s: %v", key.Namespace, key.Name, err)
+ c.deleteSecret(key)
+ }
+ return nil
+}
+
+func (c *Controller) reconcileSecret(key types.NamespacedName, secret *corev1.Secret) error {
+ clusterID, kubeconfig, err := parseRemoteClusterSecret(secret)
+ if err != nil {
+ return err
+ }
+ if clusterID == c.configClusterID {
+ return fmt.Errorf("remote cluster %q matches the config cluster", clusterID)
+ }
+
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ if existingKey, ok := c.clusterToSecret[clusterID]; ok && existingKey != key {
+ return fmt.Errorf("remote cluster %q is already registered by secret %s/%s", clusterID, existingKey.Namespace, existingKey.Name)
+ }
+ if previousID, ok := c.secretToCluster[key]; ok && previousID != clusterID {
+ c.removeClusterLocked(previousID)
+ }
+
+ kubeconfigSha := sha256.Sum256(kubeconfig)
+ if existing, ok := c.cs.Get(clusterID); ok && existing.kubeConfigSha == kubeconfigSha {
+ c.secretToCluster[key] = clusterID
+ c.clusterToSecret[clusterID] = key
+ return nil
+ }
+
+ client, err := c.newRemoteClient(kubeconfig, clusterID, c.configOverrides...)
+ if err != nil {
+ return err
+ }
+ cluster := NewRemoteCluster(clusterID, client, kubeconfigSha)
+ if existing, ok := c.cs.Get(clusterID); ok {
+ for _, handler := range c.handlers {
+ handler.clusterUpdated(cluster)
+ }
+ existing.Close()
+ } else {
+ c.handleAdd(cluster)
+ }
+ c.cs.Store(cluster)
+ c.secretToCluster[key] = clusterID
+ c.clusterToSecret[clusterID] = key
+ c.startRemoteCluster(cluster)
+ return nil
+}
+
+func (c *Controller) deleteSecret(key types.NamespacedName) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ clusterID, ok := c.secretToCluster[key]
+ if !ok {
+ return
+ }
+ c.removeClusterLocked(clusterID)
+}
+
+func (c *Controller) removeClusterLocked(clusterID cluster.ID) {
+ cluster, ok := c.cs.Delete(clusterID)
+ if !ok {
+ return
+ }
+ key := c.clusterToSecret[clusterID]
+ delete(c.clusterToSecret, clusterID)
+ delete(c.secretToCluster, key)
+ c.handleDelete(clusterID)
+ cluster.Close()
+}
+
+func parseRemoteClusterSecret(secret *corev1.Secret) (cluster.ID, []byte, error) {
+ if secret.Type != corev1.SecretTypeOpaque {
+ return "", nil, fmt.Errorf("secret type %q must be %q", secret.Type, corev1.SecretTypeOpaque)
+ }
+ clusterName := strings.TrimSpace(string(secret.Data[RemoteClusterSecretClusterNameKey]))
+ if clusterName == "" {
+ return "", nil, fmt.Errorf("missing %q", RemoteClusterSecretClusterNameKey)
+ }
+ kubeconfig := secret.Data[RemoteClusterSecretKubeconfigKey]
+ if len(kubeconfig) == 0 {
+ return "", nil, fmt.Errorf("missing %q", RemoteClusterSecretKubeconfigKey)
+ }
+ return cluster.ID(clusterName), kubeconfig, nil
+}
+
+func newRemoteClient(kubeconfig []byte, clusterID cluster.ID, configOverrides ...func(*rest.Config)) (kube.Client, error) {
+ clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
+ if err != nil {
+ return nil, fmt.Errorf("parse kubeconfig: %w", err)
+ }
+ restConfig, err := clientConfig.ClientConfig()
+ if err != nil {
+ return nil, fmt.Errorf("build kubeconfig: %w", err)
+ }
+ for _, override := range configOverrides {
+ override(restConfig)
+ }
+ restConfig = kube.SetRestDefaults(restConfig)
+ client, err := kube.NewClient(kube.NewClientConfigForRestConfig(restConfig), clusterID)
+ if err != nil {
+ return nil, err
+ }
+ return kube.EnableCrdWatcher(client), nil
+}
diff --git a/pkg/kube/multicluster/secretcontroller_test.go b/pkg/kube/multicluster/secretcontroller_test.go
new file mode 100644
index 00000000..4fa45c25
--- /dev/null
+++ b/pkg/kube/multicluster/secretcontroller_test.go
@@ -0,0 +1,178 @@
+package multicluster
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/apache/dubbo-kubernetes/pkg/cluster"
+ "github.com/apache/dubbo-kubernetes/pkg/kube"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/rest"
+)
+
+type recordingHandler struct {
+ added []cluster.ID
+ updated []cluster.ID
+ deleted []cluster.ID
+}
+
+func (h *recordingHandler) clusterAdded(cluster *Cluster) ComponentConstraint {
+ h.added = append(h.added, cluster.ID)
+ return syncedComponent{}
+}
+
+func (h *recordingHandler) clusterUpdated(cluster *Cluster) ComponentConstraint {
+ h.updated = append(h.updated, cluster.ID)
+ return syncedComponent{}
+}
+
+func (h *recordingHandler) clusterDeleted(clusterID cluster.ID) {
+ h.deleted = append(h.deleted, clusterID)
+}
+
+func (h *recordingHandler) HasSynced() bool {
+ return true
+}
+
+type syncedComponent struct{}
+
+func (syncedComponent) Close() {}
+func (syncedComponent) HasSynced() bool { return true }
+
+func TestReconcileSecretLifecycle(t *testing.T) {
+ controller := newTestController(func([]byte, cluster.ID, ...func(*rest.Config)) (kube.Client, error) {
+ return nil, nil
+ })
+ handler := &recordingHandler{}
+ controller.handlers = append(controller.handlers, handler)
+ key := types.NamespacedName{Namespace: "dubbo-system", Name: "remote-cluster1"}
+
+ if err := controller.reconcileSecret(key, remoteSecret("cluster1", "kubeconfig-a")); err != nil {
+ t.Fatalf("reconcileSecret() add error = %v", err)
+ }
+ if got := handler.added; len(got) != 1 || got[0] != "cluster1" {
+ t.Fatalf("added = %v, want [cluster1]", got)
+ }
+ if _, ok := controller.cs.Get("cluster1"); !ok {
+ t.Fatal("cluster1 not stored after add")
+ }
+
+ if err := controller.reconcileSecret(key, remoteSecret("cluster1", "kubeconfig-b")); err != nil {
+ t.Fatalf("reconcileSecret() update error = %v", err)
+ }
+ if got := handler.updated; len(got) != 1 || got[0] != "cluster1" {
+ t.Fatalf("updated = %v, want [cluster1]", got)
+ }
+
+ controller.deleteSecret(key)
+ if got := handler.deleted; len(got) != 1 || got[0] != "cluster1" {
+ t.Fatalf("deleted = %v, want [cluster1]", got)
+ }
+ if _, ok := controller.cs.Get("cluster1"); ok {
+ t.Fatal("cluster1 still stored after delete")
+ }
+}
+
+func TestReconcileSecretRejectsBadRemoteCluster(t *testing.T) {
+ controller := newTestController(func([]byte, cluster.ID, ...func(*rest.Config)) (kube.Client, error) {
+ return nil, errors.New("bad kubeconfig")
+ })
+ key := types.NamespacedName{Namespace: "dubbo-system", Name: "remote-cluster1"}
+ if err := controller.reconcileSecret(key, remoteSecret("cluster1", "kubeconfig-a")); err == nil {
+ t.Fatal("reconcileSecret() error = nil, want bad kubeconfig")
+ }
+ if _, ok := controller.cs.Get("cluster1"); ok {
+ t.Fatal("cluster1 stored despite bad kubeconfig")
+ }
+}
+
+func TestReconcileSecretRejectsDuplicateClusterID(t *testing.T) {
+ controller := newTestController(func([]byte, cluster.ID, ...func(*rest.Config)) (kube.Client, error) {
+ return nil, nil
+ })
+ if err := controller.reconcileSecret(types.NamespacedName{Namespace: "dubbo-system", Name: "remote-a"}, remoteSecret("cluster1", "kubeconfig-a")); err != nil {
+ t.Fatalf("reconcileSecret() first add error = %v", err)
+ }
+ if err := controller.reconcileSecret(types.NamespacedName{Namespace: "dubbo-system", Name: "remote-b"}, remoteSecret("cluster1", "kubeconfig-b")); err == nil {
+ t.Fatal("reconcileSecret() duplicate error = nil, want duplicate rejection")
+ }
+}
+
+func TestParseRemoteClusterSecretValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ secret *corev1.Secret
+ }{
+ {
+ name: "wrong type",
+ secret: &corev1.Secret{
+ Type: corev1.SecretTypeTLS,
+ Data: map[string][]byte{
+ RemoteClusterSecretClusterNameKey: []byte("cluster1"),
+ RemoteClusterSecretKubeconfigKey: []byte("config"),
+ },
+ },
+ },
+ {
+ name: "missing cluster name",
+ secret: &corev1.Secret{
+ Type: corev1.SecretTypeOpaque,
+ Data: map[string][]byte{
+ RemoteClusterSecretKubeconfigKey: []byte("config"),
+ },
+ },
+ },
+ {
+ name: "missing kubeconfig",
+ secret: &corev1.Secret{
+ Type: corev1.SecretTypeOpaque,
+ Data: map[string][]byte{
+ RemoteClusterSecretClusterNameKey: []byte("cluster1"),
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ if _, _, err := parseRemoteClusterSecret(test.secret); err == nil {
+ t.Fatal("parseRemoteClusterSecret() error = nil, want validation error")
+ }
+ })
+ }
+}
+
+func TestClusterStoreHasSynced(t *testing.T) {
+ store := NewClusterStore()
+ cluster1 := NewRemoteCluster("cluster1", nil, [32]byte{})
+ store.Store(cluster1)
+ if store.HasSynced() {
+ t.Fatal("HasSynced() = true before cluster sync")
+ }
+ cluster1.initialSync.Store(true)
+ if !store.HasSynced() {
+ t.Fatal("HasSynced() = false after cluster sync")
+ }
+}
+
+func newTestController(builder func([]byte, cluster.ID, ...func(*rest.Config)) (kube.Client, error)) *Controller {
+ return &Controller{
+ configClusterID: "primary",
+ cs: NewClusterStore(),
+ secretToCluster: make(map[types.NamespacedName]cluster.ID),
+ clusterToSecret: make(map[cluster.ID]types.NamespacedName),
+ newRemoteClient: builder,
+ startRemoteCluster: func(*Cluster) {},
+ }
+}
+
+func remoteSecret(clusterName, kubeconfig string) *corev1.Secret {
+ return &corev1.Secret{
+ Type: corev1.SecretTypeOpaque,
+ Data: map[string][]byte{
+ RemoteClusterSecretClusterNameKey: []byte(clusterName),
+ RemoteClusterSecretKubeconfigKey: []byte(kubeconfig),
+ },
+ }
+}
diff --git a/samples/multicluster/README.md b/samples/multicluster/README.md
new file mode 100644
index 00000000..d59a8669
--- /dev/null
+++ b/samples/multicluster/README.md
@@ -0,0 +1,99 @@
+# 多集群主从跨网
+
+## 结论
+
+`chore_v47` 进入主从跨网方案:主集群运行 `dubbod`,远端集群通过 Secret 接入主控制面;跨集群流量不再直连远端 Pod IP,而是通过各集群的 east-west `dxgate` 转发。
+
+## 前提
+
+- 主集群 API Server 能访问远端 API Server。
+- 两个集群共享同一信任根。
+- 两个集群已安装 Gateway API CRD。
+- 远端 webhook 和远端 `dxgate` 都能访问主集群控制面地址。
+- 跨网模式允许 Pod CIDR 重叠,但每个集群都必须有可被其他集群访问的 east-west gateway 地址。
+
+## 安装主控制面
+
+主集群安装时同时打开远端访问和 east-west 配置:
+
+```bash
+dubboctl manifest generate \
+ --set values.global.multicluster.remoteAccess.enabled=true \
+ --set values.global.multicluster.remoteAccess.serviceType=NodePort \
+ --set values.global.multicluster.remoteAccess.grpcPort=26010 \
+ --set values.global.multicluster.remoteAccess.certificateHosts[0]=<主集群对外地址> \
+ --set values.global.multicluster.eastWestGateway.enabled=true \
+ --set values.global.multicluster.eastWestGateway.serviceType=NodePort \
+ --set values.global.multicluster.eastWestGateway.nodePort=32443 \
+ --set values.global.multicluster.eastWestGateway.gateways[0].clusterName=remote \
+ --set values.global.multicluster.eastWestGateway.gateways[0].address=<远端 east-west 地址> \
+ --set values.global.multicluster.eastWestGateway.gateways[0].port=15443 \
+ --set values.global.multicluster.eastWestGateway.gateways[1].clusterName=Kubernetes \
+ --set values.global.multicluster.eastWestGateway.gateways[1].address=<主集群 east-west 地址> \
+ --set values.global.multicluster.eastWestGateway.gateways[1].port=15443 | kubectl --context cluster1 apply -f -
+```
+
+`DUBBO_EASTWEST_GATEWAYS` 会写入 `dubbod` Deployment。EDS 下发时,本集群 endpoint 保持 Pod IP,远端 cluster shard 会改写成对应 gateway 地址。
+
+## 接入远端集群
+
+主集群创建远端 Secret:
+
+```bash
+dubboctl multicluster create-remote-secret \
+ --cluster-name remote \
+ --kubeconfig ~/.kube/config2 \
+ --context kubernetes-admin@kubernetes | kubectl --context cluster1 apply -f -
+```
+
+远端集群安装注入 webhook:
+
+```bash
+dubboctl multicluster generate-remote-manifest \
+ --cluster-name remote \
+ --webhook-url https://<主集群远端 webhook 地址>:<端口> \
+ --xds-address <主集群远端 xDS/CA 地址>:<端口> \
+ --ca-address <主集群远端 xDS/CA 地址>:<端口> \
+ --ca-bundle-file ./ca-cert.pem | kubectl --context cluster2 apply -f -
+```
+
+远端集群创建 east-west Gateway;主控制面会通过远端 Secret 监听该 Gateway,并在远端生成 `dxgate`:
+
+```bash
+dubboctl multicluster generate-eastwest-gateway \
+ --xds-address http://<主集群可访问 ADS 地址>:<grpc-xds NodePort 或 LB 端口> \
+ --service-type NodePort \
+ --node-port 32443 | kubectl --context cluster2 apply -f -
+```
+
+## 部署样例
+
+```bash
+for ctx in cluster1 cluster2; do
+ kubectl --context "${ctx}" create ns app --dry-run=client -o yaml | kubectl --context "${ctx}" apply -f -
+ kubectl --context "${ctx}" label namespace app dubbo-injection=enabled --overwrite
+ kubectl --context "${ctx}" apply -f samples/app/deployment.yaml
+done
+
+kubectl --context cluster1 -n app scale deploy/nginx-v2 --replicas=0
+kubectl --context cluster2 -n app scale deploy/nginx-v1 --replicas=0
+kubectl --context cluster1 apply -f samples/app/meshservice.yaml
+kubectl --context cluster1 apply -f samples/multicluster/eastwest-nginx-httproute.yaml
+```
+
+`HTTPRoute` 只需要放在主控制面的配置源里;远端 `dxgate` 连接主控制面 ADS 后会拿到同一份路由配置,并把流量转给远端本地 `nginx` Service。
+
+## 验证
+
+```bash
+kubectl --context cluster2 -n app get pod -l app=nginx-consumer \
+ -o jsonpath='{.items[0].spec.containers[0].env[?(@.name=="DUBBO_META_CLUSTER_ID")].value}{"\n"}'
+
+kubectl --context cluster1 -n app exec deploy/nginx-consumer -- \
+ dubbod xclient --print-route --expect v1=50,v2=50
+
+kubectl --context cluster2 -n app exec deploy/nginx-consumer -- \
+ dubbod xclient --print-route --expect v1=50,v2=50
+```
+
+跨网模式下,`--print-route` 里远端 endpoint 应显示 east-west gateway 地址,而不是远端 Pod IP。
diff --git a/dubboinstaller/charts/dubbod/values.yaml b/samples/multicluster/eastwest-nginx-httproute.yaml
similarity index 70%
copy from dubboinstaller/charts/dubbod/values.yaml
copy to samples/multicluster/eastwest-nginx-httproute.yaml
index fcb95320..42f32ca8 100644
--- a/dubboinstaller/charts/dubbod/values.yaml
+++ b/samples/multicluster/eastwest-nginx-httproute.yaml
@@ -13,17 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-_internal_default_values_not_set:
- revision: ""
-
- global:
- proxy:
- clusterDomain: "cluster.local"
-
- statusPort: 26020
-
- gui:
- port: 26080
- nodePort: 30080
-
- configValidation: true
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ name: nginx-eastwest
+ namespace: app
+spec:
+ parentRefs:
+ - name: dubbod-eastwest-gateway
+ namespace: dubbo-system
+ sectionName: http-eastwest
+ hostnames:
+ - nginx.app.svc.cluster.local
+ rules:
+ - backendRefs:
+ - name: nginx
+ port: 80
diff --git a/tests/e2e/multicluster-primary-remote.sh b/tests/e2e/multicluster-primary-remote.sh
new file mode 100755
index 00000000..5622ce1b
--- /dev/null
+++ b/tests/e2e/multicluster-primary-remote.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+KUBECTL="${KUBECTL:-kubectl}"
+DUBBOCTL="${DUBBOCTL:-go run ./dubboctl}"
+PRIMARY_CONTEXT="${PRIMARY_CONTEXT:?PRIMARY_CONTEXT is required}"
+REMOTE_CONTEXT="${REMOTE_CONTEXT:?REMOTE_CONTEXT is required}"
+PRIMARY_KUBECONFIG="${PRIMARY_KUBECONFIG:-${KUBECONFIG:-${HOME}/.kube/config}}"
+REMOTE_CLUSTER_NAME="${REMOTE_CLUSTER_NAME:?REMOTE_CLUSTER_NAME is required}"
+REMOTE_WEBHOOK_URL="${REMOTE_WEBHOOK_URL:?REMOTE_WEBHOOK_URL is required}"
+REMOTE_XDS_ADDRESS="${REMOTE_XDS_ADDRESS:?REMOTE_XDS_ADDRESS is required}"
+REMOTE_CA_ADDRESS="${REMOTE_CA_ADDRESS:?REMOTE_CA_ADDRESS is required}"
+CA_BUNDLE_FILE="${CA_BUNDLE_FILE:?CA_BUNDLE_FILE is required}"
+REMOTE_KUBECONFIG="${REMOTE_KUBECONFIG:-${KUBECONFIG:-${HOME}/.kube/config}}"
+MULTICLUSTER_NETWORK_MODE="${MULTICLUSTER_NETWORK_MODE:-cross-network}"
+PRIMARY_CLUSTER_NAME="${PRIMARY_CLUSTER_NAME:-Kubernetes}"
+EASTWEST_GATEWAY_PORT="${EASTWEST_GATEWAY_PORT:-15443}"
+EASTWEST_SERVICE_TYPE="${EASTWEST_SERVICE_TYPE:-NodePort}"
+EASTWEST_NODE_PORT="${EASTWEST_NODE_PORT:-32443}"
+REMOTE_EASTWEST_XDS_ADDRESS="${REMOTE_EASTWEST_XDS_ADDRESS:-}"
+
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" cluster-info >/dev/null
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" cluster-info >/dev/null
+
+if [[ "${MULTICLUSTER_NETWORK_MODE}" == "same-network" ]]; then
+ primary_pod_cidrs="$("${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" get nodes \
+ -o jsonpath='{range .items[*]}{.spec.podCIDR}{"\n"}{end}' | sort -u)"
+ remote_pod_cidrs="$("${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" get nodes \
+ -o jsonpath='{range .items[*]}{.spec.podCIDR}{"\n"}{end}' | sort -u)"
+ if comm -12 <(printf '%s\n' "${primary_pod_cidrs}") <(printf '%s\n' "${remote_pod_cidrs}") | grep -q .; then
+ echo "primary and remote pod CIDRs overlap; same-network multicluster requires disjoint, routable Pod CIDRs" >&2
+ exit 1
+ fi
+else
+ primary_gateways="$("${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" -n dubbo-system get deploy dubbod \
+ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="DUBBO_EASTWEST_GATEWAYS")].value}')"
+ if [[ "${primary_gateways}" != *"${REMOTE_CLUSTER_NAME}="* || "${primary_gateways}" != *"${PRIMARY_CLUSTER_NAME}="* ]]; then
+ echo "dubbod DUBBO_EASTWEST_GATEWAYS=${primary_gateways}; cross-network requires ${REMOTE_CLUSTER_NAME}=... and ${PRIMARY_CLUSTER_NAME}=..." >&2
+ exit 1
+ fi
+ if [[ -z "${REMOTE_EASTWEST_XDS_ADDRESS}" ]]; then
+ echo "REMOTE_EASTWEST_XDS_ADDRESS is required for cross-network remote dxgate bootstrap" >&2
+ exit 1
+ fi
+fi
+
+${DUBBOCTL} multicluster create-remote-secret \
+ --cluster-name "${REMOTE_CLUSTER_NAME}" \
+ --kubeconfig "${REMOTE_KUBECONFIG}" \
+ --context "${REMOTE_CONTEXT}" | "${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" apply -f -
+
+${DUBBOCTL} multicluster generate-remote-manifest \
+ --cluster-name "${REMOTE_CLUSTER_NAME}" \
+ --webhook-url "${REMOTE_WEBHOOK_URL}" \
+ --xds-address "${REMOTE_XDS_ADDRESS}" \
+ --ca-address "${REMOTE_CA_ADDRESS}" \
+ --ca-bundle-file "${CA_BUNDLE_FILE}" | "${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" apply -f -
+
+if [[ "${MULTICLUSTER_NETWORK_MODE}" != "same-network" ]]; then
+ gateway_args=(
+ multicluster generate-eastwest-gateway
+ --xds-address "${REMOTE_EASTWEST_XDS_ADDRESS}"
+ --service-type "${EASTWEST_SERVICE_TYPE}"
+ --port "${EASTWEST_GATEWAY_PORT}"
+ )
+ if [[ "${EASTWEST_NODE_PORT}" != "0" ]]; then
+ gateway_args+=(--node-port "${EASTWEST_NODE_PORT}")
+ fi
+ ${DUBBOCTL} "${gateway_args[@]}" | "${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" apply -f -
+fi
+
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" create namespace app --dry-run=client -o yaml | \
+ "${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" apply -f -
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" label namespace app dubbo-injection=enabled --overwrite
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" apply -f samples/app/deployment.yaml
+
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" create namespace app --dry-run=client -o yaml | \
+ "${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" apply -f -
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" label namespace app dubbo-injection=enabled --overwrite
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" apply -f samples/app/deployment.yaml
+
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" -n app scale deploy/nginx-v2 --replicas=0
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" -n app scale deploy/nginx-v1 --replicas=0
+
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" -n app rollout status deploy/nginx-v1 --timeout=180s
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" -n app rollout status deploy/nginx-v2 --timeout=180s
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" -n app rollout status deploy/nginx-consumer --timeout=180s
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" -n app rollout status deploy/nginx-v1 --timeout=180s
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" -n app rollout status deploy/nginx-v2 --timeout=180s
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" -n app rollout status deploy/nginx-consumer --timeout=180s
+
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" apply -f samples/app/meshservice.yaml
+if [[ "${MULTICLUSTER_NETWORK_MODE}" != "same-network" ]]; then
+ "${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" apply -f samples/multicluster/eastwest-nginx-httproute.yaml
+fi
+
+remote_cluster_id="$("${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" -n app get pod -l app=nginx-consumer \
+ -o jsonpath='{.items[0].spec.containers[0].env[?(@.name=="DUBBO_META_CLUSTER_ID")].value}')"
+if [[ "${remote_cluster_id}" != "${REMOTE_CLUSTER_NAME}" ]]; then
+ echo "remote consumer cluster id = ${remote_cluster_id}, want ${REMOTE_CLUSTER_NAME}" >&2
+ exit 1
+fi
+
+"${KUBECTL}" --kubeconfig "${PRIMARY_KUBECONFIG}" --context "${PRIMARY_CONTEXT}" -n app exec deploy/nginx-consumer -- \
+ dubbod xclient --expect v1=50,v2=50 100 | sort | uniq -c
+"${KUBECTL}" --kubeconfig "${REMOTE_KUBECONFIG}" --context "${REMOTE_CONTEXT}" -n app exec deploy/nginx-consumer -- \
+ dubbod xclient --expect v1=50,v2=50 100 | sort | uniq -c
diff --git a/tools/release_notes/generate_release_notes.py b/tools/release_notes/generate_release_notes.py
new file mode 100644
index 00000000..1db7afbd
--- /dev/null
+++ b/tools/release_notes/generate_release_notes.py
@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import datetime as dt
+import os
+import re
+import subprocess
+from pathlib import Path
+from typing import Iterable
+
+
+VERSION_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$")
+PR_RE = re.compile(r"\(#(?P<num>\d+)\)")
+
+
+@dataclasses.dataclass(frozen=True)
+class Commit:
+ subject: str
+ sha: str
+ author: str
+
+
+@dataclasses.dataclass(frozen=True)
+class ReleaseContext:
+ tag: str
+ version: str
+ previous_tag: str | None
+ repo: str
+ date: dt.date
+
+
+CATEGORY_ORDER = [
+ ("security", "Security", "安全"),
+ ("features", "Features", "新增功能"),
+ ("traffic", "Traffic Management", "流量管理"),
+ ("installation", "Installation and CLI", "安装与命令行"),
+ ("fixes", "Bug Fixes", "问题修复"),
+ ("tests", "Tests", "测试"),
+ ("docs", "Documentation", "文档"),
+ ("other", "Other Changes", "其他变更"),
+]
+
+
+def run_git(args: list[str], cwd: Path) -> str:
+ return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip()
+
+
+def parse_version(tag: str) -> tuple[int, int, int]:
+ match = VERSION_RE.match(strip_ref(tag))
+ if not match:
+ raise ValueError(f"tag must be a semantic version such as 0.4.0 or v0.4.0: {tag}")
+ return tuple(int(part) for part in match.groups())
+
+
+def strip_ref(tag: str) -> str:
+ return tag.removeprefix("refs/tags/")
+
+
+def display_version(tag: str) -> str:
+ return strip_ref(tag).removeprefix("v")
+
+
+def minor_series(version: str) -> str:
+ major, minor, _ = parse_version(version)
+ return f"{major}.{minor}.x"
+
+
+def english_date(value: dt.date) -> str:
+ months = [
+ "Jan",
+ "Feb",
+ "Mar",
+ "Apr",
+ "May",
+ "Jun",
+ "Jul",
+ "Aug",
+ "Sep",
+ "Oct",
+ "Nov",
+ "Dec",
+ ]
+ return f"{months[value.month - 1]} {value.day}, {value.year}"
+
+
+def release_kind(version: str) -> tuple[str, str]:
+ _, _, patch = parse_version(version)
+ if patch == 0:
+ return "minor release", "小版本发布"
+ return "patch release", "补丁版本发布"
+
+
+def source_changes_url(ctx: ReleaseContext) -> str:
+ base = f"https://github.com/{ctx.repo}"
+ if ctx.previous_tag:
+ return f"{base}/compare/{ctx.previous_tag}...{ctx.tag}"
+ return f"{base}/commits/{ctx.tag}"
+
+
+def github_release_url(ctx: ReleaseContext) -> str:
+ return f"https://github.com/{ctx.repo}/releases/tag/{ctx.tag}"
+
+
+def previous_tag_for(tag: str, cwd: Path) -> str | None:
+ current = parse_version(tag)
+ tags = []
+ for candidate in run_git(["tag", "--list"], cwd).splitlines():
+ try:
+ parsed = parse_version(candidate)
+ except ValueError:
+ continue
+ if parsed < current:
+ tags.append((parsed, candidate))
+ if not tags:
+ return None
+ tags.sort()
+ return tags[-1][1]
+
+
+def git_log_range(ctx: ReleaseContext) -> str:
+ if ctx.previous_tag:
+ return f"{ctx.previous_tag}..{ctx.tag}"
+ return ctx.tag
+
+
+def collect_commits(ctx: ReleaseContext, cwd: Path) -> list[Commit]:
+ fmt = "%s%x1f%h%x1f%an"
+ range_ref = git_log_range(ctx)
+ output = run_git(["log", "--no-merges", f"--pretty=format:{fmt}", range_ref], cwd)
+ if not output:
+ output = run_git(["log", f"--pretty=format:{fmt}", range_ref], cwd)
+ commits = []
+ for line in output.splitlines():
+ parts = line.split("\x1f")
+ if len(parts) == 3:
+ commits.append(Commit(subject=parts[0], sha=parts[1], author=parts[2]))
+ return [commit for commit in commits if not is_release_marker(commit.subject)]
+
+
+def is_release_marker(subject: str) -> bool:
+ normalized = subject.strip().lower()
+ return bool(
+ re.fullmatch(r"release[-: ]?v?\d+\.\d+\.\d+", normalized)
+ or re.fullmatch(r"version v?\d+\.\d+\.\d+.*", normalized)
+ or normalized in {"fix version", "bump version", "version bump"}
+ )
+
+
+def classify(subject: str) -> str:
+ value = subject.lower()
+ if any(token in value for token in ["cve", "security", "auth", "jwt", "tls", "mtls", "rbac"]):
+ return "security"
+ if any(token in value for token in ["feat", "feature", "add ", "added", "new "]):
+ return "features"
+ if any(
+ token in value
+ for token in [
+ "gateway",
+ "xds",
+ "route",
+ "traffic",
+ "meshservice",
+ "multicluster",
+ "east-west",
+ "proxyless",
+ "eds",
+ "lds",
+ "rds",
+ ]
+ ):
+ return "traffic"
+ if any(token in value for token in ["helm", "chart", "install", "dubboctl", "operator", "manifest"]):
+ return "installation"
+ if any(token in value for token in ["fix", "bug", "repair", "resolve", "correct"]):
+ return "fixes"
+ if any(token in value for token in ["test", "e2e", "coverage"]):
+ return "tests"
+ if any(token in value for token in ["doc", "readme", "website"]):
+ return "docs"
+ return "other"
+
+
+def clean_subject(subject: str) -> str:
+ value = subject.strip()
+ value = re.sub(r"^(feat|fix|docs|test|chore|refactor|perf)(\([^)]+\))?:\s*", "", value, flags=re.I)
+ return value[:1].upper() + value[1:] if value else subject
+
+
+def format_commit(commit: Commit, repo: str) -> str:
+ subject = clean_subject(commit.subject)
+ match = PR_RE.search(subject)
+ if match:
+ pr = match.group("num")
+ subject = PR_RE.sub("", subject).strip()
+ return f"{subject} ([#{pr}](https://github.com/{repo}/pull/{pr}))"
+ return f"{subject} ({commit.sha})"
+
+
+def grouped_changes(commits: Iterable[Commit], repo: str) -> dict[str, list[str]]:
+ groups: dict[str, list[str]] = {key: [] for key, _, _ in CATEGORY_ORDER}
+ seen: set[str] = set()
+ for commit in commits:
+ item = format_commit(commit, repo)
+ if item in seen:
+ continue
+ seen.add(item)
+ groups[classify(commit.subject)].append(item)
+ return {key: values for key, values in groups.items() if values}
+
+
+def render_changes(groups: dict[str, list[str]], language: str) -> str:
+ if not groups:
+ return "- No user-facing changes were detected in the commit range.\n" if language == "en" else "- 未检测到面向用户的变更。\n"
+ lines: list[str] = []
+ for key, en_title, zh_title in CATEGORY_ORDER:
+ items = groups.get(key)
+ if not items:
+ continue
+ title = en_title if language == "en" else zh_title
+ lines.append(f"### {title}")
+ lines.append("")
+ for item in items:
+ lines.append(f"- {item}")
+ lines.append("")
+ return "\n".join(lines).rstrip() + "\n"
+
+
+def render_release_body(ctx: ReleaseContext, commits: list[Commit]) -> str:
+ en_kind, _ = release_kind(ctx.version)
+ groups = grouped_changes(commits, ctx.repo)
+ previous = f"Kdubbo {display_version(ctx.previous_tag)}" if ctx.previous_tag else "the initial release"
+ return f"""# Announcing Kdubbo {ctx.version}
+
+Kdubbo {ctx.version} {en_kind}.
+
+{english_date(ctx.date)}
+
+This release note describes what is different between {previous} and Kdubbo {ctx.version}.
+
+[Before you upgrade](https://kdubbo.github.io/operation/) | [Download]({github_release_url(ctx)}) | [Docs](https://kdubbo.github.io/) | [Source Changes]({source_changes_url(ctx)})
+
+## Changes
+
+{render_changes(groups, "en")}"""
+
+
+def render_docs_page(ctx: ReleaseContext, commits: list[Commit], language: str) -> str:
+ en_kind, zh_kind = release_kind(ctx.version)
+ groups = grouped_changes(commits, ctx.repo)
+ if language == "en":
+ previous = f"Kdubbo {display_version(ctx.previous_tag)}" if ctx.previous_tag else "the initial release"
+ return f"""# Announcing Kdubbo {ctx.version}
+
+Kdubbo {ctx.version} {en_kind}.
+
+{english_date(ctx.date)}
+
+This release note describes what is different between {previous} and Kdubbo {ctx.version}.
+
+[Before you upgrade](../../../operation/) | [Download]({github_release_url(ctx)}) | [Docs](https://kdubbo.github.io/) | [Source Changes]({source_changes_url(ctx)})
+
+## Changes
+
+{render_changes(groups, "en")}"""
+
+ previous_zh = f"Kdubbo {display_version(ctx.previous_tag)}" if ctx.previous_tag else "初始版本"
+ return f"""# Kdubbo {ctx.version} 发布公告
+
+Kdubbo {ctx.version} {zh_kind}。
+
+发布日期:{ctx.date.isoformat()}
+
+本发布说明描述 {previous_zh} 和 Kdubbo {ctx.version} 之间的差异。
+
+[升级前注意事项](../../../operation/) | [下载]({github_release_url(ctx)}) | [文档](https://kdubbo.github.io/) | [源码变更]({source_changes_url(ctx)})
+
+## 变更
+
+{render_changes(groups, "zh")}"""
+
+
+def docs_paths(docs_root: Path, version: str) -> tuple[Path, Path]:
+ base = docs_root / "latest" / "release" / minor_series(version)
+ return base / f"announcing-{version}.md", base / f"announcing-{version}.en.md"
+
+
+def render_index_entry(ctx: ReleaseContext, commits: list[Commit]) -> str:
+ groups = grouped_changes(commits, ctx.repo)
+ first_items = []
+ for key, _, zh_title in CATEGORY_ORDER:
+ if key in groups:
+ first_items.append(f"{zh_title}:{groups[key][0]}")
+ if len(first_items) == 3:
+ break
+ summary = "\n".join(f" - {item}" for item in first_items) or " - 查看完整发布说明"
+ page = f"{minor_series(ctx.version)}/announcing-{ctx.version}.md"
+ return f""" ### [{ctx.version}]({page})
+
+ 发布日期:{ctx.date.isoformat()}
+
+{summary}
+
+ [查看完整发布说明]({page})
+ [查看 GitHub Release]({github_release_url(ctx)})
+
+ ---"""
+
+
+def update_docs_index(docs_root: Path, ctx: ReleaseContext, commits: list[Commit]) -> None:
+ index = docs_root / "latest" / "release" / "index.md"
+ start = " <!-- KDUBBO_RELEASE_NOTES:START -->"
+ end = " <!-- KDUBBO_RELEASE_NOTES:END -->"
+ entry = render_index_entry(ctx, commits)
+ content = index.read_text(encoding="utf-8")
+ if start in content and end in content:
+ before, rest = content.split(start, 1)
+ block, after = rest.split(end, 1)
+ old_entries = [part.strip("\n") for part in block.strip().split("\n\n ---\n\n") if part.strip()]
+ old_entries = [part for part in old_entries if f"announcing-{ctx.version}.md" not in part]
+ normalized = [entry.rstrip()]
+ for part in old_entries:
+ normalized.append(part.rstrip() + "\n\n ---")
+ new_block = "\n\n" + "\n\n".join(normalized).rstrip() + "\n\n"
+ index.write_text(before + start + new_block + end + after, encoding="utf-8")
+ return
+
+ anchor = '=== "发布公告"'
+ if anchor not in content:
+ raise ValueError(f"{index} does not contain the release announcement tab")
+ replacement = f'{anchor}\n\n{start}\n\n{entry}\n\n{end}\n'
+ index.write_text(content.replace(anchor, replacement, 1), encoding="utf-8")
+
+
+def write_docs(docs_root: Path, ctx: ReleaseContext, commits: list[Commit]) -> None:
+ zh_path, en_path = docs_paths(docs_root, ctx.version)
+ zh_path.parent.mkdir(parents=True, exist_ok=True)
+ zh_path.write_text(render_docs_page(ctx, commits, "zh"), encoding="utf-8")
+ en_path.write_text(render_docs_page(ctx, commits, "en"), encoding="utf-8")
+ update_docs_index(docs_root, ctx, commits)
+
+
+def build_context(args: argparse.Namespace, cwd: Path) -> ReleaseContext:
+ tag = strip_ref(args.tag or os.environ.get("GITHUB_REF_NAME", ""))
+ if not tag:
+ tag = strip_ref(run_git(["describe", "--tags", "--abbrev=0"], cwd))
+ version = display_version(tag)
+ parse_version(version)
+ previous_tag = args.previous_tag if args.previous_tag is not None else previous_tag_for(tag, cwd)
+ date_value = dt.date.fromisoformat(args.date) if args.date else dt.date.today()
+ repo = args.repo or os.environ.get("GITHUB_REPOSITORY", "apache/dubbo-kubernetes")
+ return ReleaseContext(tag=tag, version=version, previous_tag=previous_tag, repo=repo, date=date_value)
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Generate Kdubbo release notes and kdubbo.github.io pages.")
+ parser.add_argument("--tag", help="Release tag, for example 0.4.0 or v0.4.0.")
+ parser.add_argument("--previous-tag", help="Previous release tag. Auto-detected from git tags when omitted.")
+ parser.add_argument("--repo", help="GitHub repository, for example apache/dubbo-kubernetes.")
+ parser.add_argument("--date", help="Release date in YYYY-MM-DD format. Defaults to today.")
+ parser.add_argument("--release-body", type=Path, help="Write the GitHub Release body markdown to this file.")
+ parser.add_argument("--docs-root", type=Path, help="Path to kdubbo.github.io/docs to update.")
+ parser.add_argument("--print", action="store_true", help="Print the GitHub Release body to stdout.")
+ return parser.parse_args()
+
+
+def main() -> int:
+ args = parse_args()
+ cwd = Path.cwd()
+ ctx = build_context(args, cwd)
+ commits = collect_commits(ctx, cwd)
+ body = render_release_body(ctx, commits)
+ if args.release_body:
+ args.release_body.parent.mkdir(parents=True, exist_ok=True)
+ args.release_body.write_text(body, encoding="utf-8")
+ if args.docs_root:
+ write_docs(args.docs_root, ctx, commits)
+ if args.print:
+ print(body)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/release_notes/generate_release_notes_test.py b/tools/release_notes/generate_release_notes_test.py
new file mode 100644
index 00000000..8e5a2d7b
--- /dev/null
+++ b/tools/release_notes/generate_release_notes_test.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import datetime as dt
+import importlib.util
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+
+MODULE_PATH = Path(__file__).with_name("generate_release_notes.py")
+SPEC = importlib.util.spec_from_file_location("generate_release_notes", MODULE_PATH)
+assert SPEC and SPEC.loader
+release_notes = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = release_notes
+SPEC.loader.exec_module(release_notes)
+
+
+class ReleaseNotesTest(unittest.TestCase):
+ def context(self):
+ return release_notes.ReleaseContext(
+ tag="0.4.1",
+ version="0.4.1",
+ previous_tag="0.4.0",
+ repo="apache/dubbo-kubernetes",
+ date=dt.date(2026, 5, 18),
+ )
+
+ def commits(self):
+ return [
+ release_notes.Commit("feat: Add multicluster east-west gateway (#910)", "abc1234", "dev"),
+ release_notes.Commit("fix: Repair remote webhook CA bundle (#911)", "def5678", "dev"),
+ release_notes.Commit("docs: Update install guide", "abcd999", "dev"),
+ ]
+
+ def test_release_body_uses_istio_style_sections(self):
+ body = release_notes.render_release_body(self.context(), self.commits())
+
+ self.assertIn("# Announcing Kdubbo 0.4.1", body)
+ self.assertIn("Kdubbo 0.4.1 patch release.", body)
+ self.assertIn("May 18, 2026", body)
+ self.assertIn("[Before you upgrade]", body)
+ self.assertIn("[Source Changes](https://github.com/apache/dubbo-kubernetes/compare/0.4.0...0.4.1)", body)
+ self.assertIn("### Features", body)
+ self.assertIn("[#910](https://github.com/apache/dubbo-kubernetes/pull/910)", body)
+
+ def test_writes_bilingual_docs_and_index_entry(self):
+ with tempfile.TemporaryDirectory() as temp:
+ docs = Path(temp) / "docs"
+ release_dir = docs / "latest" / "release"
+ release_dir.mkdir(parents=True)
+ (release_dir / "index.md").write_text('# 公告栏\n\n=== "发布公告"\n\n ## 0.3.x\n', encoding="utf-8")
+
+ release_notes.write_docs(docs, self.context(), self.commits())
+
+ zh = release_dir / "0.4.x" / "announcing-0.4.1.md"
+ en = release_dir / "0.4.x" / "announcing-0.4.1.en.md"
+ index = release_dir / "index.md"
+
+ self.assertTrue(zh.exists())
+ self.assertTrue(en.exists())
+ self.assertIn("# Kdubbo 0.4.1 发布公告", zh.read_text(encoding="utf-8"))
+ self.assertIn("# Announcing Kdubbo 0.4.1", en.read_text(encoding="utf-8"))
+ self.assertIn("KDUBBO_RELEASE_NOTES:START", index.read_text(encoding="utf-8"))
+ self.assertIn("[0.4.1](0.4.x/announcing-0.4.1.md)", index.read_text(encoding="utf-8"))
+
+
+if __name__ == "__main__":
+ unittest.main()