You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2026/05/21 08:18:06 UTC
(airavata-custos) 02/03: Wait for connector goroutines to drain on shutdown
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch access-integration-v3
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
commit 8b4a0b478f0bafc321453cbb37d8a4b7aff5e7e4
Author: lahiruj <la...@gmail.com>
AuthorDate: Thu May 21 03:37:21 2026 -0400
Wait for connector goroutines to drain on shutdown
---
cmd/server/main.go | 20 +++++++++++++++++++-
connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go | 20 ++++++++++----------
.../SLURM/Association-Mapper/pkg/smapper/loader.go | 3 ++-
internal/connectors/loader.go | 7 ++++---
4 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/cmd/server/main.go b/cmd/server/main.go
index 6c3a96037..14a954b30 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -26,6 +26,7 @@ import (
"os"
"os/signal"
"strconv"
+ "sync"
"syscall"
"time"
@@ -79,7 +80,10 @@ func run() error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
- if err := connectors.LoadConnectors(ctx, database, eventBus, svc); err != nil {
+ // Tracks every background goroutine spawned by connectors so we can wait
+ // for them to drain on shutdown instead of killing them mid-flight.
+ var connectorsWG sync.WaitGroup
+ if err := connectors.LoadConnectors(ctx, database, eventBus, svc, &connectorsWG); err != nil {
return err
}
@@ -117,6 +121,20 @@ func run() error {
if err := httpServer.Shutdown(shutdownCtx); err != nil {
return err
}
+
+ slog.Info("waiting for connectors to drain")
+ connectorsDone := make(chan struct{})
+ go func() {
+ connectorsWG.Wait()
+ close(connectorsDone)
+ }()
+ select {
+ case <-connectorsDone:
+ slog.Info("connectors drained cleanly")
+ case <-time.After(30 * time.Second):
+ slog.Warn("connector drain timed out; some workers may have leaked")
+ }
+
slog.Info("server stopped cleanly")
return nil
}
diff --git a/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go b/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
index e6dc0d5d5..7dcc52c7d 100644
--- a/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
+++ b/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
@@ -43,10 +43,9 @@ import (
const connectorName = "amie"
-// LoadConnector applies the connector's migrations and starts its background
-// workers. It returns nil and skips when AMIE_BASE_URL / AMIE_SITE_CODE /
+// LoadConnector skips silently when AMIE_BASE_URL / AMIE_SITE_CODE /
// AMIE_API_KEY are not all set.
-func LoadConnector(ctx context.Context, database *sqlx.DB, eventBus *events.Bus, coreService *coreservice.Service) error {
+func LoadConnector(ctx context.Context, database *sqlx.DB, eventBus *events.Bus, coreService *coreservice.Service, wg *sync.WaitGroup) error {
cfg := loadConfig()
if cfg.AMIE.APIKey == "" || cfg.AMIE.BaseURL == "" || cfg.AMIE.SiteCode == "" {
slog.Warn("AMIE credentials not fully provided, skipping AMIE connector")
@@ -92,15 +91,16 @@ func LoadConnector(ctx context.Context, database *sqlx.DB, eventBus *events.Bus,
poller := worker.NewPoller(amie, packetStore, eventStore, met, database, cfg.AMIE)
processor := worker.NewProcessor(eventStore, packetStore, errorStore, router, met, database, cfg.AMIE)
- var wg sync.WaitGroup
wg.Add(2)
- go func() { defer wg.Done(); poller.Run(ctx) }()
- go func() { defer wg.Done(); processor.Run(ctx) }()
-
go func() {
- <-ctx.Done()
- wg.Wait()
- slog.Info("AMIE connector stopped")
+ defer wg.Done()
+ poller.Run(ctx)
+ slog.Info("AMIE poller stopped")
+ }()
+ go func() {
+ defer wg.Done()
+ processor.Run(ctx)
+ slog.Info("AMIE processor stopped")
}()
slog.Info("AMIE connector started", "site", cfg.AMIE.SiteCode, "baseURL", cfg.AMIE.BaseURL)
diff --git a/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go b/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go
index 7efd4b29a..9eee2ae74 100644
--- a/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go
+++ b/connectors/SLURM/Association-Mapper/pkg/smapper/loader.go
@@ -4,6 +4,7 @@ import (
"context"
"log/slog"
"os"
+ "sync"
"github.com/jmoiron/sqlx"
@@ -13,7 +14,7 @@ import (
"github.com/apache/airavata-custos/pkg/service"
)
-func LoadConnector(_ context.Context, _ *sqlx.DB, eventBus *events.Bus, coreService *service.Service) error {
+func LoadConnector(_ context.Context, _ *sqlx.DB, eventBus *events.Bus, coreService *service.Service, _ *sync.WaitGroup) error {
// Read url, username, and password from environment variables
apiUrl := os.Getenv("SLURM_API")
diff --git a/internal/connectors/loader.go b/internal/connectors/loader.go
index 0632fd7eb..99fb221c0 100644
--- a/internal/connectors/loader.go
+++ b/internal/connectors/loader.go
@@ -20,6 +20,7 @@ package connectors
import (
"context"
"log/slog"
+ "sync"
"github.com/jmoiron/sqlx"
@@ -29,17 +30,17 @@ import (
"github.com/apache/airavata-custos/pkg/service"
)
-func LoadConnectors(ctx context.Context, database *sqlx.DB, eventBus *events.Bus, coreService *service.Service) error {
+func LoadConnectors(ctx context.Context, database *sqlx.DB, eventBus *events.Bus, coreService *service.Service, wg *sync.WaitGroup) error {
slog.Info("loading connectors")
slog.Info("loading SLURM Association Mapper connector")
- if err := smapper.LoadConnector(ctx, database, eventBus, coreService); err != nil {
+ if err := smapper.LoadConnector(ctx, database, eventBus, coreService, wg); err != nil {
slog.Error("failed to load SLURM Association Mapper connector", "error", err)
return err
}
slog.Info("loading AMIE connector")
- if err := amie.LoadConnector(ctx, database, eventBus, coreService); err != nil {
+ if err := amie.LoadConnector(ctx, database, eventBus, coreService, wg); err != nil {
slog.Error("failed to load AMIE connector", "error", err)
return err
}