You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/08/28 01:06:35 UTC
(skywalking-banyandb) branch accesslog created (now b27e088e)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch accesslog
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
at b27e088e Enhance access log functionality with sampling option and add tests
This branch includes the following new commits:
new b27e088e Enhance access log functionality with sampling option and add tests
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(skywalking-banyandb) 01/01: Enhance access log functionality with sampling option and add tests
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch accesslog
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit b27e088e2748f1ad614852bbf813632a64d00795
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Aug 28 01:05:08 2025 +0000
Enhance access log functionality with sampling option and add tests
This commit modifies the access log implementation to include a sampling option, allowing for configurable request handling based on the `sampled` parameter.
---
banyand/liaison/grpc/measure.go | 4 +-
banyand/liaison/grpc/server.go | 20 +++--
banyand/liaison/grpc/stream.go | 4 +-
pkg/accesslog/file.go | 181 +++++++++++++++++++++++++++++-----------
pkg/accesslog/file_test.go | 167 ++++++++++++++++++++++++++++++++++++
5 files changed, 314 insertions(+), 62 deletions(-)
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index 5d17d72b..373a676c 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -57,9 +57,9 @@ func (ms *measureService) setLogger(log *logger.Logger) {
ms.l = log
}
-func (ms *measureService) activeIngestionAccessLog(root string) (err error) {
+func (ms *measureService) activeIngestionAccessLog(root string, sampled bool) (err error) {
if ms.ingestionAccessLog, err = accesslog.
- NewFileLog(root, "measure-ingest-%s", 10*time.Minute, ms.log); err != nil {
+ NewFileLog(root, "measure-ingest-%s", 10*time.Minute, ms.log, sampled); err != nil {
return err
}
return nil
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
index fae1180b..0a3eb307 100644
--- a/banyand/liaison/grpc/server.go
+++ b/banyand/liaison/grpc/server.go
@@ -81,9 +81,9 @@ type server struct {
databasev1.UnimplementedSnapshotServiceServer
omr observability.MetricsRegistry
schemaRepo metadata.Repo
- *topNAggregationRegistryServer
- *groupRegistryServer
- stopCh chan struct{}
+ *indexRuleBindingRegistryServer
+ groupRepo *groupRepo
+ stopCh chan struct{}
*indexRuleRegistryServer
*measureRegistryServer
streamSVC *streamService
@@ -94,22 +94,23 @@ type server struct {
ser *grpclib.Server
tlsReloader *pkgtls.Reloader
*propertyServer
- *indexRuleBindingRegistryServer
+ *topNAggregationRegistryServer
+ *groupRegistryServer
*traceRegistryServer
authReloader *auth.Reloader
- groupRepo *groupRepo
metrics *metrics
- certFile string
keyFile string
authConfigFile string
host string
addr string
accessLogRootPath string
+ certFile string
accessLogRecorders []accessLogRecorder
maxRecvMsgSize run.Bytes
port uint32
- enableIngestionAccessLog bool
tls bool
+ enableIngestionAccessLog bool
+ accessLogSampled bool
healthAuthEnabled bool
}
@@ -198,7 +199,7 @@ func (s *server) PreRun(_ context.Context) error {
if s.enableIngestionAccessLog {
for _, alr := range s.accessLogRecorders {
- if err := alr.activeIngestionAccessLog(s.accessLogRootPath); err != nil {
+ if err := alr.activeIngestionAccessLog(s.accessLogRootPath, s.accessLogSampled); err != nil {
return err
}
}
@@ -258,6 +259,7 @@ func (s *server) FlagSet() *run.FlagSet {
fs.Uint32Var(&s.port, "grpc-port", 17912, "the port of banyand listens")
fs.BoolVar(&s.enableIngestionAccessLog, "enable-ingestion-access-log", false, "enable ingestion access log")
fs.StringVar(&s.accessLogRootPath, "access-log-root-path", "", "access log root path")
+ fs.BoolVar(&s.accessLogSampled, "access-log-sampled", true, "if true, requests may be dropped when the channel is full; if false, requests are never dropped")
fs.DurationVar(&s.streamSVC.writeTimeout, "stream-write-timeout", 15*time.Second, "timeout for writing stream among liaison nodes")
fs.DurationVar(&s.measureSVC.writeTimeout, "measure-write-timeout", 15*time.Second, "timeout for writing measure among liaison nodes")
fs.DurationVar(&s.measureSVC.maxWaitDuration, "measure-metadata-cache-wait-duration", 0,
@@ -400,6 +402,6 @@ func (s *server) GracefulStop() {
}
type accessLogRecorder interface {
- activeIngestionAccessLog(root string) error
+ activeIngestionAccessLog(root string, sampled bool) error
Close() error
}
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 5f9949e3..b9a6236c 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -57,9 +57,9 @@ func (s *streamService) setLogger(log *logger.Logger) {
s.l = log
}
-func (s *streamService) activeIngestionAccessLog(root string) (err error) {
+func (s *streamService) activeIngestionAccessLog(root string, sampled bool) (err error) {
if s.ingestionAccessLog, err = accesslog.
- NewFileLog(root, "stream-ingest-%s", 10*time.Minute, s.log); err != nil {
+ NewFileLog(root, "stream-ingest-%s", 10*time.Minute, s.log, sampled); err != nil {
return err
}
return nil
diff --git a/pkg/accesslog/file.go b/pkg/accesslog/file.go
index b03b2deb..a30e19ae 100644
--- a/pkg/accesslog/file.go
+++ b/pkg/accesslog/file.go
@@ -18,6 +18,7 @@
package accesslog
import (
+ "bytes"
"fmt"
"os"
"path"
@@ -30,60 +31,45 @@ import (
"github.com/apache/skywalking-banyandb/pkg/run"
)
+const (
+ // DefaultBatchSize is the default number of requests to batch before writing to the file.
+ DefaultBatchSize = 100
+ // DefaultFlushInterval is the default interval at which to flush the batch to the file.
+ DefaultFlushInterval = 1 * time.Second
+)
+
type fileLog struct {
- file *os.File
+ file *os.File // Single file for all goroutines
validRequests chan proto.Message
closer *run.Closer
+ sampled bool
}
// NewFileLog creates a new file access log.
-func NewFileLog(root, template string, interval time.Duration, log *logger.Logger) (Log, error) {
- var file *os.File
- var err error
- if file, err = createFile(root, template); err != nil {
- return nil, err
+// sampled: if true (default), requests may be dropped when the channel is full.
+// If false, requests are never dropped but use buffered channel to prevent blocking.
+func NewFileLog(root, template string, interval time.Duration, log *logger.Logger, sampled bool) (Log, error) {
+ var validRequests chan proto.Message
+ if sampled {
+ validRequests = make(chan proto.Message, 100)
+ } else {
+ // For non-sampled mode, use buffered channel to prevent blocking on writes
+ validRequests = make(chan proto.Message, 1000) // Buffer to handle burst writes
}
+
f := &fileLog{
- validRequests: make(chan proto.Message, 100),
- file: file,
+ validRequests: validRequests,
closer: run.NewCloser(1),
+ sampled: sampled,
}
- go func() {
- defer f.closer.Done()
- defer close(f.validRequests)
-
- ticker := time.NewTicker(interval)
-
- for {
- select {
- case <-f.closer.CloseNotify():
- if f.file != nil {
- f.file.Close()
- }
- ticker.Stop()
- return
- case <-ticker.C:
- if f.file != nil {
- f.file.Close()
- }
- if f.file, err = createFile(root, template); err != nil {
- log.Error().Err(err).Msg("failed to open file for writing")
- continue
- }
- case req := <-f.validRequests:
- if f.file != nil {
- data, err := protojson.Marshal(req)
- if err != nil {
- log.Error().Err(err).Msg("failed to marshal request")
- continue
- }
- if _, err := f.file.WriteString(string(data) + "\n"); err != nil {
- log.Error().Err(err).Msg("failed to write request to file")
- }
- }
- }
- }
- }()
+
+ // Create single file for both sampled and non-sampled modes
+ var err error
+ if f.file, err = createFile(root, template); err != nil {
+ return nil, err
+ }
+
+ go startConsumer(f, root, template, interval, log)
return f, nil
}
@@ -91,10 +77,17 @@ func (f *fileLog) Write(req proto.Message) error {
if f == nil {
return nil
}
- select {
- case f.validRequests <- req:
- default:
- return fmt.Errorf("access log is full")
+
+ if f.sampled {
+ // Sampled mode: may drop requests if channel is full
+ select {
+ case f.validRequests <- req:
+ default:
+ return fmt.Errorf("access log is full")
+ }
+ } else {
+ // Non-sampled mode: never drop requests, block until buffer has space
+ f.validRequests <- req
}
return nil
}
@@ -104,10 +97,100 @@ func (f *fileLog) Close() error {
return nil
}
f.closer.CloseThenWait()
+
+ if f.file != nil {
+ f.file.Close()
+ f.file = nil
+ }
+
+ // Close the channel after all consumers are done
+ close(f.validRequests)
return nil
}
+// startConsumer starts a consumer goroutine that handles file rotation and request processing.
+func startConsumer(f *fileLog, root, template string, interval time.Duration, log *logger.Logger) {
+ defer f.closer.Done()
+
+ rotationTicker := time.NewTicker(interval)
+ defer rotationTicker.Stop()
+
+ flushTicker := time.NewTicker(DefaultFlushInterval)
+ defer flushTicker.Stop()
+
+ batch := make([]proto.Message, 0, DefaultBatchSize)
+
+ for {
+ select {
+ case <-f.closer.CloseNotify():
+ // Before closing, flush any remaining requests in the batch.
+ flushBatch(f.file, batch, log)
+ return
+ case <-rotationTicker.C:
+ flushBatch(f.file, batch, log)
+ clear(batch)
+ rotateFile(root, template, f, log)
+ case <-flushTicker.C:
+ if len(batch) > 0 {
+ flushBatch(f.file, batch, log)
+ clear(batch)
+ }
+ case req, ok := <-f.validRequests:
+ if !ok {
+ // Channel closed, flush any remaining requests and exit.
+ flushBatch(f.file, batch, log)
+ return
+ }
+ batch = append(batch, req)
+ if len(batch) >= DefaultBatchSize {
+ flushBatch(f.file, batch, log)
+ clear(batch)
+ }
+ }
+ }
+}
+
+// flushBatch marshals and writes a batch of requests to the specified file.
+func flushBatch(file *os.File, batch []proto.Message, log *logger.Logger) {
+ if file == nil || len(batch) == 0 {
+ return
+ }
+
+ var buffer bytes.Buffer
+ for _, req := range batch {
+ data, err := protojson.Marshal(req)
+ if err != nil {
+ log.Error().Err(err).Msg("failed to marshal request")
+ continue
+ }
+ buffer.Write(data)
+ buffer.WriteString("\n")
+ }
+
+ if _, err := file.Write(buffer.Bytes()); err != nil {
+ log.Error().Err(err).Msg("failed to write requests to file")
+ }
+}
+
+// rotateFile handles the common logic for file rotation with single file.
+func rotateFile(root, template string, f *fileLog, log *logger.Logger) {
+ // Close current file
+ if f.file != nil {
+ f.file.Close()
+ f.file = nil
+ }
+
+ // Create new file
+ newFile, err := createFile(root, template)
+ if err != nil {
+ log.Error().Err(err).Msg("failed to open file for writing")
+ return
+ }
+ f.file = newFile
+}
+
func createFile(root string, template string) (*os.File, error) {
- fileName := path.Join(root, fmt.Sprintf(template, time.Now().Format("20060102_150405")))
+ timestamp := time.Now().Format("20060102_150405")
+ fileName := path.Join(root, fmt.Sprintf(template, timestamp))
return os.Create(fileName)
}
diff --git a/pkg/accesslog/file_test.go b/pkg/accesslog/file_test.go
new file mode 100644
index 00000000..60ddd40c
--- /dev/null
+++ b/pkg/accesslog/file_test.go
@@ -0,0 +1,167 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package accesslog
+
+import (
+ "bufio"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/types/known/wrapperspb"
+
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+const testTemplate = "access-%s.log"
+
+func initTestLogger(t *testing.T) {
+ t.Helper()
+ err := logger.Init(logger.Logging{Env: "dev", Level: "debug"})
+ require.NoError(t, err)
+}
+
+func newFileLogForTest(t *testing.T, root string, rotationInterval time.Duration, sampled bool) Log {
+ t.Helper()
+ initTestLogger(t)
+ l := logger.GetLogger("test", "accesslog")
+ flog, err := NewFileLog(root, testTemplate, rotationInterval, l, sampled)
+ require.NoError(t, err)
+ return flog
+}
+
+func listLogFiles(t *testing.T, dir string) []string {
+ t.Helper()
+ entries, err := os.ReadDir(dir)
+ require.NoError(t, err)
+ out := make([]string, 0, len(entries))
+ for _, e := range entries {
+ if e.IsDir() {
+ continue
+ }
+ out = append(out, filepath.Join(dir, e.Name()))
+ }
+ return out
+}
+
+func countLines(t *testing.T, filePath string) int {
+ t.Helper()
+ f, err := os.Open(filePath)
+ require.NoError(t, err)
+ defer f.Close()
+ s := bufio.NewScanner(f)
+ lines := 0
+ for s.Scan() {
+ if len(s.Bytes()) == 0 {
+ continue
+ }
+ lines++
+ }
+ require.NoError(t, s.Err())
+ return lines
+}
+
+func totalLinesInDir(t *testing.T, dir string) int {
+ t.Helper()
+ total := 0
+ for _, f := range listLogFiles(t, dir) {
+ total += countLines(t, f)
+ }
+ return total
+}
+
+func writeMessages(t *testing.T, l Log, n int) {
+ t.Helper()
+ for i := 0; i < n; i++ {
+ // Use a simple well-known protobuf message
+ msg := wrapperspb.String("msg")
+ require.NoError(t, l.Write(msg))
+ }
+}
+
+func waitForLines(t *testing.T, dir string, want int, within time.Duration) {
+ t.Helper()
+ require.Eventually(t, func() bool {
+ return totalLinesInDir(t, dir) == want
+ }, within, 20*time.Millisecond)
+}
+
+func TestFileLog_SimpleSampled(t *testing.T) {
+ dir := t.TempDir()
+ flog := newFileLogForTest(t, dir, 10*time.Second, true)
+ defer flog.Close()
+
+ writeMessages(t, flog, 10)
+
+ // Rely on periodic flush (DefaultFlushInterval)
+ waitForLines(t, dir, 10, 3*time.Second)
+}
+
+func TestFileLog_Unsampled(t *testing.T) {
+ dir := t.TempDir()
+ flog := newFileLogForTest(t, dir, 10*time.Second, false)
+ defer flog.Close()
+
+ writeMessages(t, flog, 10)
+ waitForLines(t, dir, 10, 3*time.Second)
+}
+
+func TestFileLog_FlushBatchOnThreshold(t *testing.T) {
+ dir := t.TempDir()
+ flog := newFileLogForTest(t, dir, 10*time.Second, true)
+ defer flog.Close()
+
+ // Write exactly one batch; should flush immediately without waiting for flush ticker
+ writeMessages(t, flog, DefaultBatchSize)
+
+ // Expect lines to appear quickly since flush occurs on reaching batch size
+ require.Eventually(t, func() bool {
+ files := listLogFiles(t, dir)
+ if len(files) == 0 {
+ return false
+ }
+ return countLines(t, files[0]) == DefaultBatchSize
+ }, 1*time.Second, 20*time.Millisecond)
+}
+
+func TestFileLog_Rotation(t *testing.T) {
+ dir := t.TempDir()
+ // Use 1s rotation interval; we'll manually invoke rotation after crossing second boundary
+ flog := newFileLogForTest(t, dir, 1*time.Second, true)
+ defer flog.Close()
+
+ // Write a full batch to force immediate flush
+ writeMessages(t, flog, DefaultBatchSize)
+ require.Eventually(t, func() bool {
+ return totalLinesInDir(t, dir) == DefaultBatchSize
+ }, 2*time.Second, 20*time.Millisecond)
+
+ // Ensure we cross a second boundary to get a new timestamped filename, then rotate explicitly
+ time.Sleep(1100 * time.Millisecond)
+ rotateFile(dir, testTemplate, flog.(*fileLog), logger.GetLogger("test"))
+
+ // Write more after rotation (again full batch to force flush)
+ writeMessages(t, flog, DefaultBatchSize)
+
+ // Eventually we should have lines across at least 2 files
+ require.Eventually(t, func() bool {
+ return len(listLogFiles(t, dir)) >= 2
+ }, 3*time.Second, 20*time.Millisecond)
+}