You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/12/08 06:23:08 UTC

(skywalking-banyandb) branch main updated: internal TLS reload (#882)

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git


The following commit(s) were added to refs/heads/main by this push:
     new 8c647e99 internal TLS reload (#882)
8c647e99 is described below

commit 8c647e999b4f31f26de94007a384b1f45fb15b4d
Author: OmCheeLin <19...@163.com>
AuthorDate: Mon Dec 8 14:23:02 2025 +0800

    internal TLS reload (#882)
    
    
    
    ---------
    
    Co-authored-by: 吴晟 Wu Sheng <wu...@foxmail.com>
---
 banyand/queue/pub/pub.go    | 100 +++++++++++++++++++++++++++++++++++++++++++-
 banyand/queue/sub/server.go |  50 +++++++++++++++++++---
 docs/operation/security.md  |  22 ++++++++--
 3 files changed, 161 insertions(+), 11 deletions(-)

diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index fd69d37a..26d5320f 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -30,6 +30,7 @@ import (
 	"go.uber.org/multierr"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 
@@ -47,6 +48,7 @@ import (
 	"github.com/apache/skywalking-banyandb/pkg/grpchelper"
 	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/run"
+	pkgtls "github.com/apache/skywalking-banyandb/pkg/tls"
 )
 
 // ChunkedSyncClientConfig configures chunked sync client behavior.
@@ -75,6 +77,7 @@ type pub struct {
 	writableProbe   map[string]map[string]struct{}
 	cbStates        map[string]*circuitState
 	caCertPath      string
+	caCertReloader  *pkgtls.Reloader
 	prefix          string
 	retryPolicy     string
 	allowedRoles    []databasev1.Role
@@ -100,7 +103,7 @@ func (p *pub) FlagSet() *run.FlagSet {
 func (p *pub) Validate() error {
 	// simple sanity‑check: if TLS is on, a CA bundle must be provided
 	if p.tlsEnabled && p.caCertPath == "" {
-		return fmt.Errorf("TLS is enabled (--internal-tls), but no CA certificate file was provided (--internal-ca-cert is required)")
+		return fmt.Errorf("TLS is enabled (--data-client-tls), but no CA certificate file was provided (--data-client-ca-cert is required)")
 	}
 	return nil
 }
@@ -110,6 +113,11 @@ func (p *pub) Register(topic bus.Topic, handler schema.EventHandler) {
 }
 
 func (p *pub) GracefulStop() {
+	// Stop CA certificate reloader if enabled
+	if p.caCertReloader != nil {
+		p.caCertReloader.Stop()
+	}
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	for i := range p.evictable {
@@ -126,6 +134,35 @@ func (p *pub) GracefulStop() {
 
 // Serve implements run.Service.
 func (p *pub) Serve() run.StopNotify {
+	// Start CA certificate reloader if enabled
+	if p.caCertReloader != nil {
+		if err := p.caCertReloader.Start(); err != nil {
+			p.log.Error().Err(err).Msg("Failed to start CA certificate reloader")
+			stopCh := p.closer.CloseNotify()
+			return stopCh
+		}
+		p.log.Info().Str("caCertPath", p.caCertPath).Msg("Started CA certificate file monitoring")
+
+		// Listen for certificate update events
+		certUpdateCh := p.caCertReloader.GetUpdateChannel()
+		stopCh := p.closer.CloseNotify()
+		if p.closer.AddRunning() {
+			go func() {
+				defer p.closer.Done()
+				for {
+					select {
+					case <-certUpdateCh:
+						p.log.Info().Msg("CA certificate updated, reconnecting clients")
+						p.reconnectAllClients()
+					case <-stopCh:
+						return
+					}
+				}
+			}()
+		}
+		return stopCh
+	}
+
 	return p.closer.CloseNotify()
 }
 
@@ -332,6 +369,17 @@ func (p *pub) PreRun(context.Context) error {
 	}
 
 	p.log = logger.GetLogger("server-queue-pub-" + p.prefix)
+
+	// Initialize CA certificate reloader if TLS is enabled and CA cert path is provided
+	if p.tlsEnabled && p.caCertPath != "" {
+		var err error
+		p.caCertReloader, err = pkgtls.NewClientCertReloader(p.caCertPath, p.log)
+		if err != nil {
+			return errors.Wrapf(err, "failed to initialize CA certificate reloader for %s", p.prefix)
+		}
+		p.log.Info().Str("caCertPath", p.caCertPath).Msg("Initialized CA certificate reloader")
+	}
+
 	return nil
 }
 
@@ -441,6 +489,23 @@ func isFailoverError(err error) bool {
 }
 
 func (p *pub) getClientTransportCredentials() ([]grpc.DialOption, error) {
+	if !p.tlsEnabled {
+		return grpchelper.SecureOptions(nil, false, false, "")
+	}
+
+	// Use reloader if available (for dynamic reloading)
+	if p.caCertReloader != nil {
+		// Extract server name from the connection (we'll use a default for now)
+		// The actual server name will be validated by the TLS handshake
+		tlsConfig, err := p.caCertReloader.GetClientTLSConfig("")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get TLS config from reloader: %w", err)
+		}
+		creds := credentials.NewTLS(tlsConfig)
+		return []grpc.DialOption{grpc.WithTransportCredentials(creds)}, nil
+	}
+
+	// Fallback to static file reading if reloader is not available
 	opts, err := grpchelper.SecureOptions(nil, p.tlsEnabled, false, p.caCertPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load TLS config: %w", err)
@@ -448,6 +513,39 @@ func (p *pub) getClientTransportCredentials() ([]grpc.DialOption, error) {
 	return opts, nil
 }
 
+// reconnectAllClients reconnects all active clients when CA certificate is updated.
+func (p *pub) reconnectAllClients() {
+	// Collect nodes and close connections
+	p.mu.Lock()
+	nodesToReconnect := make([]schema.Metadata, 0, len(p.registered))
+	for name, node := range p.registered {
+		// Handle evictable nodes: close channel and remove from evictable
+		if en, ok := p.evictable[name]; ok {
+			close(en.c)
+			delete(p.evictable, name)
+		}
+		// Handle active nodes: close connection and remove from active
+		if client, ok := p.active[name]; ok {
+			_ = client.conn.Close()
+			delete(p.active, name)
+			p.deleteClient(client.md)
+		}
+		md := schema.Metadata{
+			TypeMeta: schema.TypeMeta{
+				Kind: schema.KindNode,
+			},
+			Spec: node,
+		}
+		nodesToReconnect = append(nodesToReconnect, md)
+	}
+	p.mu.Unlock()
+
+	// Reconnect with new credentials
+	for _, md := range nodesToReconnect {
+		p.OnAddOrUpdate(md)
+	}
+}
+
 // NewChunkedSyncClient implements queue.Client.
 func (p *pub) NewChunkedSyncClient(node string, chunkSize uint32) (queue.ChunkedSyncClient, error) {
 	return p.NewChunkedSyncClientWithConfig(node, &ChunkedSyncClientConfig{
diff --git a/banyand/queue/sub/server.go b/banyand/queue/sub/server.go
index c1949b20..05ca5507 100644
--- a/banyand/queue/sub/server.go
+++ b/banyand/queue/sub/server.go
@@ -51,6 +51,7 @@ import (
 	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/meter"
 	"github.com/apache/skywalking-banyandb/pkg/run"
+	pkgtls "github.com/apache/skywalking-banyandb/pkg/tls"
 )
 
 const defaultRecvSize = 10 << 20
@@ -72,6 +73,7 @@ type server struct {
 	streamv1.UnimplementedStreamServiceServer
 	databasev1.UnimplementedSnapshotServiceServer
 	creds               credentials.TransportCredentials
+	tlsReloader         *pkgtls.Reloader
 	omr                 observability.MetricsRegistry
 	metrics             *metrics
 	ser                 *grpclib.Server
@@ -126,6 +128,17 @@ func NewServerWithPorts(omr observability.MetricsRegistry, flagNamePrefix string
 func (s *server) PreRun(_ context.Context) error {
 	s.log = logger.GetLogger("server-queue-sub")
 	s.metrics = newMetrics(s.omr.With(queueSubScope))
+
+	// Initialize TLS reloader if TLS is enabled
+	if s.tls {
+		var err error
+		s.tlsReloader, err = pkgtls.NewReloader(s.certFile, s.keyFile, s.log)
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize TLS reloader for queue server")
+		}
+		s.log.Info().Str("certFile", s.certFile).Str("keyFile", s.keyFile).Msg("Initialized TLS reloader for queue server")
+	}
+
 	return nil
 }
 
@@ -188,18 +201,37 @@ func (s *server) Validate() error {
 	if s.keyFile == "" {
 		return errServerKey
 	}
-	creds, errTLS := credentials.NewServerTLSFromFile(s.certFile, s.keyFile)
-	if errTLS != nil {
-		return errors.Wrap(errTLS, "failed to load cert and key")
-	}
-	s.creds = creds
+	// TLS reloader will be initialized in PreRun, so we don't need to load credentials here
+	// The credentials will be loaded dynamically when the server starts
 	return nil
 }
 
 func (s *server) Serve() run.StopNotify {
 	var opts []grpclib.ServerOption
 	if s.tls {
-		opts = []grpclib.ServerOption{grpclib.Creds(s.creds)}
+		// Start TLS reloader if enabled
+		if s.tlsReloader != nil {
+			if err := s.tlsReloader.Start(); err != nil {
+				s.log.Error().Err(err).Msg("Failed to start TLS reloader for queue server")
+				stopCh := make(chan struct{})
+				close(stopCh)
+				return stopCh
+			}
+			s.log.Info().Str("certFile", s.certFile).Str("keyFile", s.keyFile).Msg("Started TLS file monitoring for queue server")
+			tlsConfig := s.tlsReloader.GetTLSConfig()
+			creds := credentials.NewTLS(tlsConfig)
+			opts = []grpclib.ServerOption{grpclib.Creds(creds)}
+		} else {
+			// Fallback to static loading if reloader is not available
+			creds, errTLS := credentials.NewServerTLSFromFile(s.certFile, s.keyFile)
+			if errTLS != nil {
+				s.log.Error().Err(errTLS).Msg("Failed to load TLS credentials")
+				stopCh := make(chan struct{})
+				close(stopCh)
+				return stopCh
+			}
+			opts = []grpclib.ServerOption{grpclib.Creds(creds)}
+		}
 	}
 	grpcPanicRecoveryHandler := func(p any) (err error) {
 		s.log.Error().Interface("panic", p).Str("stack", string(debug.Stack())).Msg("recovered from panic")
@@ -290,6 +322,12 @@ func (s *server) Serve() run.StopNotify {
 
 func (s *server) GracefulStop() {
 	s.log.Info().Msg("stopping")
+
+	// Stop TLS reloader if enabled
+	if s.tlsReloader != nil {
+		s.tlsReloader.Stop()
+	}
+
 	stopped := make(chan struct{})
 	s.clientCloser()
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
diff --git a/docs/operation/security.md b/docs/operation/security.md
index 748e8dd4..59ad6c7e 100644
--- a/docs/operation/security.md
+++ b/docs/operation/security.md
@@ -123,19 +123,33 @@ BanyanDB supports enabling TLS for the internal gRPC queue between liaison and d
 
 The following flags are used to configure internal TLS:
 
-- `--internal-tls`: Enable TLS on the internal queue client inside Liaison; if false, the queue uses plain TCP.
-- `--internal-ca-cert <path>`: PEM‑encoded CA (or bundle) that the queue client uses to verify Data‑Node server certificates.
+- `--data-client-tls`: Enable TLS on the internal queue client inside Liaison; if false, the queue uses plain TCP.
+- `--data-client-ca-cert`: PEM‑encoded CA (or bundle) that the queue client uses to verify Data‑Node server certificates.
 
 Each Liaison/Data process still advertises its certificate with the public flags (`--tls`, `--cert-file`, `--key-file`). The same certificate/key pair can be reused for both external traffic and the internal queue.
 
 **Example: Enable internal TLS between liaison and data nodes**
 
 ```shell
-banyand liaison --internal-tls=true --internal-ca-cert=ca.crt --tls=true --cert-file=server.crt --key-file=server.key
+banyand liaison --data-client-tls=true --data-client-ca-cert=ca.crt --tls=true --cert-file=server.crt --key-file=server.key
 banyand data --tls=true --cert-file=server.crt --key-file=server.key
 ```
 
-> Note: The `--internal-ca-cert` should point to the CA certificate used to sign the data node's server certificate.
+> Note: 
+> - The `--data-client-ca-cert` should point to the CA certificate used to sign the data node's server certificate.
+> - Data nodes act as servers and do not need a CA certificate to connect to liaison nodes (liaison nodes connect to data nodes, not vice versa).
+> - The flag names use the prefix "data" because liaison nodes connect to data nodes. The actual flag names are `--data-client-tls` and `--data-client-ca-cert`.
+
+**Dynamic Certificate Reloading**
+
+All certificates used for internal TLS can be reloaded automatically when they are updated:
+
+- **Liaison nodes**:
+  - CA certificate file (`--data-client-ca-cert`): Can be updated, and the server will automatically reload it and reconnect all clients to data nodes with the new certificate.
+  - Server certificate files (`--cert-file`, `--key-file`): Can be updated, and the server will automatically reload them. These certificates are used for both external client connections and can be reused for internal queue communication.
+- **Data nodes**: The server certificate files (`--cert-file`, `--key-file`) can be updated, and the server will automatically reload them without requiring a restart.
+
+You can update the files or recreate the files, and the servers will automatically reload them.
 
 ## Authorization