You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2025/09/05 21:19:38 UTC
(iceberg-go) branch main updated: feat(table): Add cleanup orphan files features (#525)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 8d84100 feat(table): Add cleanup orphan files features (#525)
8d84100 is described below
commit 8d84100a226854306329150dd85d58ce9c936947
Author: Dao Thanh Tung <tt...@accountancy.smu.edu.sg>
AuthorDate: Fri Sep 5 22:19:33 2025 +0100
feat(table): Add cleanup orphan files features (#525)
Hi team,
I believe the `DeleteOrphanFiles` functionality is a valuable addition
toward making iceberg-go more feature-complete. This implementation was
drafted with reference to the official [Iceberg Java API
specification](https://iceberg.apache.org/javadoc/1.2.0/org/apache/iceberg/actions/DeleteOrphanFiles.html).
The PR also includes corresponding unit and integration tests to ensure
reliability. Below is some example usages:
```
// Example usage:
// Dry run to see what would be deleted
result, err := table.DeleteOrphanFiles(ctx,
WithDryRun(true),
WithOlderThan(time.Now().AddDate(0, 0, -7)),
)
// Actually delete orphan files with parallel execution
result, err = table.DeleteOrphanFiles(ctx,
WithOlderThan(time.Now().AddDate(0, 0, -3)),
WithMaxConcurrency(8),
)
// Advanced usage with scheme equivalence and prefix mismatch handling
result, err = table.DeleteOrphanFiles(ctx,
WithLocation("s3://my-bucket/table/data"),
WithOlderThan(time.Now().AddDate(0, 0, -7)),
WithEqualSchemes(map[string]string{"s3,s3a,s3n": "s3"}),
WithPrefixMismatchMode(PrefixMismatchIgnore),
WithDeleteFunc(customDeleteFunction),
)
```
---------
Signed-off-by: dttung2905 <tt...@accountancy.smu.edu.sg>
---
io/blob.go | 5 +-
io/io.go | 2 +-
table/orphan_cleanup.go | 663 +++++++++++++++++++++++++++++++
table/orphan_cleanup_integration_test.go | 432 ++++++++++++++++++++
table/orphan_cleanup_test.go | 470 ++++++++++++++++++++++
5 files changed, 1569 insertions(+), 3 deletions(-)
diff --git a/io/blob.go b/io/blob.go
index 0b09d5d..32de455 100644
--- a/io/blob.go
+++ b/io/blob.go
@@ -75,6 +75,7 @@ type blobFileIO struct {
*blob.Bucket
bucketName string
+ scheme string
ctx context.Context
}
@@ -154,8 +155,8 @@ func (io *blobFileIO) NewWriter(ctx context.Context, path string, overwrite bool
nil
}
-func createBlobFS(ctx context.Context, bucket *blob.Bucket, bucketName string) IO {
- return &blobFileIO{Bucket: bucket, bucketName: bucketName, ctx: ctx}
+func createBlobFS(ctx context.Context, bucket *blob.Bucket, bucketName, scheme string) IO {
+ return &blobFileIO{Bucket: bucket, bucketName: bucketName, scheme: scheme, ctx: ctx}
}
type blobWriteFile struct {
diff --git a/io/io.go b/io/io.go
index 248dfed..508edab 100644
--- a/io/io.go
+++ b/io/io.go
@@ -267,7 +267,7 @@ func inferFileIOFromSchema(ctx context.Context, path string, props map[string]st
return nil, fmt.Errorf("IO for file '%s' not implemented", path)
}
- return createBlobFS(ctx, bucket, parsed.Host), nil
+ return createBlobFS(ctx, bucket, parsed.Host, parsed.Scheme), nil
}
// LoadFS takes a map of properties and an optional URI location
diff --git a/table/orphan_cleanup.go b/table/orphan_cleanup.go
new file mode 100644
index 0000000..b18e1ba
--- /dev/null
+++ b/table/orphan_cleanup.go
@@ -0,0 +1,663 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ stdfs "io/fs"
+ "net/url"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "strings"
+ "sync"
+ "time"
+
+ iceio "github.com/apache/iceberg-go/io"
+)
+
+// PrefixMismatchMode defines how to handle cases where candidate files have different
+// URI schemes or authorities compared to table location during orphan cleanup.
+// This is useful when files may be referenced using different but equivalent schemes
+// (e.g., s3:// vs s3a:// vs s3n://) or when cleaning up files across different locations.
+type PrefixMismatchMode int
+
+const (
+ // PrefixMismatchError causes cleanup to fail with an error when candidate files
+ // have URI schemes/authorities that don't match the table location and are not
+ // covered by configured equivalences. This is the safest default behavior.
+ PrefixMismatchError PrefixMismatchMode = iota // default
+
+ // PrefixMismatchIgnore skips candidate files that have mismatched URI schemes/authorities
+ // without treating it as an error. Files are silently ignored and not considered for deletion.
+ PrefixMismatchIgnore
+
+ // PrefixMismatchDelete treats candidate files with mismatched URI schemes/authorities
+ // as orphans and includes them for deletion. Use with caution as this may delete
+ // files from unexpected locations.
+ PrefixMismatchDelete
+)
+
+func (p PrefixMismatchMode) String() string {
+ switch p {
+ case PrefixMismatchError:
+ return "ERROR"
+ case PrefixMismatchIgnore:
+ return "IGNORE"
+ case PrefixMismatchDelete:
+ return "DELETE"
+ default:
+ return "UNKNOWN"
+ }
+}
+
+// OrphanCleanupConfig holds configuration for orphan file cleanup operations.
+type orphanCleanupConfig struct {
+ location string
+ olderThan time.Duration
+ dryRun bool
+ deleteFunc func(string) error
+ maxConcurrency int
+ prefixMismatchMode PrefixMismatchMode
+ equalSchemes map[string]string
+ equalAuthorities map[string]string
+}
+
+type OrphanCleanupOption func(*orphanCleanupConfig)
+
+func WithLocation(location string) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ cfg.location = location
+ }
+}
+
+func WithFilesOlderThan(duration time.Duration) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ cfg.olderThan = duration
+ }
+}
+
+func WithDryRun(enabled bool) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ cfg.dryRun = enabled
+ }
+}
+
+// WithDeleteFunc sets a custom delete function. If not provided, the table's FileIO
+// delete method will be used.
+func WithDeleteFunc(deleteFunc func(string) error) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ cfg.deleteFunc = deleteFunc
+ }
+}
+
+// WithMaxConcurrency sets the maximum number of goroutines for parallel deletion.
+// Defaults to a reasonable number based on the system. Only used when deleteFunc is nil or when
+// the FileIO doesn't support bulk operations.
+func WithMaxConcurrency(maxWorkers int) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ if maxWorkers > 0 {
+ cfg.maxConcurrency = maxWorkers
+ }
+ }
+}
+
+// WithPrefixMismatchMode sets how to handle situations when metadata references files
+// that match listed files except for authority/scheme differences.
+func WithPrefixMismatchMode(mode PrefixMismatchMode) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ cfg.prefixMismatchMode = mode
+ }
+}
+
+// WithEqualSchemes specifies schemes that should be considered equivalent.
+// For example, map["s3,s3a,s3n"] = "s3" treats all S3 scheme variants as equivalent.
+// The key can be a comma-separated list of schemes that map to the value scheme.
+func WithEqualSchemes(schemes map[string]string) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ if cfg.equalSchemes == nil {
+ cfg.equalSchemes = make(map[string]string)
+ }
+ for k, v := range schemes {
+ cfg.equalSchemes[k] = v
+ }
+ }
+}
+
+// WithEqualAuthorities specifies authorities that should be considered equivalent.
+// For example, map["endpoint1.s3.amazonaws.com,endpoint2.s3.amazonaws.com"] = "s3.amazonaws.com"
+// treats different S3 endpoints as equivalent. The key can be a comma-separated list.
+func WithEqualAuthorities(authorities map[string]string) OrphanCleanupOption {
+ return func(cfg *orphanCleanupConfig) {
+ if cfg.equalAuthorities == nil {
+ cfg.equalAuthorities = make(map[string]string)
+ }
+ for k, v := range authorities {
+ cfg.equalAuthorities[k] = v
+ }
+ }
+}
+
+type OrphanCleanupResult struct {
+ OrphanFileLocations []string
+ DeletedFiles []string
+ TotalSizeBytes int64
+}
+
+func (t Table) DeleteOrphanFiles(ctx context.Context, opts ...OrphanCleanupOption) (OrphanCleanupResult, error) {
+ cfg := &orphanCleanupConfig{
+ location: "", // empty means use table's data location
+ olderThan: 72 * time.Hour, // 3 days ago
+ dryRun: false,
+ deleteFunc: nil,
+ maxConcurrency: runtime.GOMAXPROCS(0), // default to number of CPUs
+ prefixMismatchMode: PrefixMismatchError, // default to safest mode
+ equalSchemes: nil, // no scheme equivalence by default
+ equalAuthorities: nil, // no authority equivalence by default
+ }
+
+ // Apply functional options
+ for _, opt := range opts {
+ opt(cfg)
+ }
+
+ return t.executeOrphanCleanup(ctx, cfg)
+}
+
+func (t Table) executeOrphanCleanup(ctx context.Context, cfg *orphanCleanupConfig) (OrphanCleanupResult, error) {
+ fs, err := t.fsF(ctx)
+ if err != nil {
+ return OrphanCleanupResult{}, fmt.Errorf("failed to get filesystem: %w", err)
+ }
+
+ scanLocation := cfg.location
+ if scanLocation == "" {
+ scanLocation = t.metadata.Location()
+ }
+
+ referencedFiles, err := t.getReferencedFiles(fs)
+ if err != nil {
+ return OrphanCleanupResult{}, fmt.Errorf("failed to get referenced files: %w", err)
+ }
+
+ allFiles, totalSize, err := t.scanFiles(fs, scanLocation, cfg)
+ if err != nil {
+ return OrphanCleanupResult{}, fmt.Errorf("failed to scan files: %w", err)
+ }
+
+ orphanFiles, err := identifyOrphanFiles(allFiles, referencedFiles, cfg)
+ if err != nil {
+ return OrphanCleanupResult{}, fmt.Errorf("failed to identify orphan files: %w", err)
+ }
+
+ result := OrphanCleanupResult{
+ OrphanFileLocations: orphanFiles,
+ TotalSizeBytes: totalSize,
+ }
+
+ if cfg.dryRun {
+ return result, nil
+ }
+ deletedFiles, err := deleteFiles(fs, orphanFiles, cfg)
+ if err != nil {
+ return OrphanCleanupResult{}, fmt.Errorf("failed to delete orphan files: %w", err)
+ }
+
+ result.DeletedFiles = deletedFiles
+
+ return result, nil
+}
+
+// getReferencedFiles collects all files referenced by current snapshots
+func (t Table) getReferencedFiles(fs iceio.IO) (map[string]bool, error) {
+ referenced := make(map[string]bool)
+ metadata := t.metadata
+
+ for entry := range metadata.PreviousFiles() {
+ referenced[entry.MetadataFile] = true
+ }
+ referenced[t.metadataLocation] = true
+
+ // Add version hint file (for Hadoop-style tables)
+ // Following Java's ReachableFileUtil.versionHintLocation() logic:
+ versionHintPath := filepath.Join(metadata.Location(), "metadata", "version-hint.text")
+ referenced[versionHintPath] = true
+
+ // TODO: Add statistics files support once iceberg-go exposes statisticsFiles()
+
+ for _, snapshot := range metadata.Snapshots() {
+ if snapshot.ManifestList != "" {
+ referenced[snapshot.ManifestList] = true
+ }
+
+ manifestFiles, err := snapshot.Manifests(fs)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read manifests for snapshot %d: %w", snapshot.SnapshotID, err)
+ }
+
+ for _, manifest := range manifestFiles {
+ referenced[manifest.FilePath()] = true
+
+ entries, err := manifest.FetchEntries(fs, false)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read manifest entries: %w", err)
+ }
+
+ for _, entry := range entries {
+ referenced[entry.DataFile().FilePath()] = true
+ }
+ }
+ }
+
+ return referenced, nil
+}
+
+func (t Table) scanFiles(fs iceio.IO, location string, cfg *orphanCleanupConfig) ([]string, int64, error) {
+ var allFiles []string
+ var totalSize int64
+
+ err := walkDirectory(fs, location, func(path string, info stdfs.FileInfo) error {
+ if info.IsDir() {
+ return nil
+ }
+
+ cutOffTime := time.Now().Add(-cfg.olderThan)
+ if !info.ModTime().Before(cutOffTime) {
+ return nil // Skip files that are newer than or equal to the threshold
+ }
+
+ allFiles = append(allFiles, path)
+ totalSize += info.Size()
+
+ return nil
+ })
+ if err != nil {
+ return nil, 0, err
+ }
+
+ return allFiles, totalSize, nil
+}
+
+// getBucket gets the Bucket field from blob storage - absolute minimal approach
+func getBucketName(fsys iceio.IO) stdfs.FS {
+ v := reflect.ValueOf(fsys).Elem() // We know it's a pointer to struct
+
+ return v.FieldByName("Bucket").Interface().(stdfs.FS)
+}
+
+// makeFileWalkFunc creates a WalkDirFunc that processes only files with path transformation
+func makeFileWalkFunc(fn func(path string, info stdfs.FileInfo) error, pathTransform func(string) string) stdfs.WalkDirFunc {
+ return func(path string, d stdfs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ info, err := d.Info()
+ if err != nil {
+ return err
+ }
+
+ return fn(pathTransform(path), info)
+ }
+}
+
+func walkDirectory(fsys iceio.IO, root string, fn func(path string, info stdfs.FileInfo) error) error {
+ switch v := fsys.(type) {
+ case iceio.LocalFS:
+ cleanRoot := strings.TrimPrefix(root, "file://")
+ if cleanRoot == "" {
+ cleanRoot = "."
+ }
+
+ return filepath.WalkDir(cleanRoot, makeFileWalkFunc(fn, func(path string) string {
+ return path
+ }))
+
+ default:
+ // For blob storage: direct field access since we know the structure
+ bucket := getBucketName(v)
+
+ parsed, err := url.Parse(root)
+ if err != nil {
+ return fmt.Errorf("invalid URL %s: %w", root, err)
+ }
+
+ walkPath := strings.TrimPrefix(parsed.Path, "/")
+ if walkPath == "" {
+ walkPath = "."
+ }
+
+ // URL transform - reconstruct full URL path
+ return stdfs.WalkDir(bucket, walkPath, makeFileWalkFunc(fn, func(path string) string {
+ return parsed.Scheme + "://" + parsed.Host + "/" + path
+ }))
+ }
+}
+
+func identifyOrphanFiles(allFiles []string, referencedFiles map[string]bool, cfg *orphanCleanupConfig) ([]string, error) {
+ normalizedReferencedFiles := make(map[string]string)
+ for refPath := range referencedFiles {
+ normalizedPath := normalizeFilePath(refPath, cfg)
+ normalizedReferencedFiles[normalizedPath] = refPath
+ // Also include the original path for direct lookup
+ normalizedReferencedFiles[refPath] = refPath
+ }
+
+ var orphans []string
+
+ for _, file := range allFiles {
+ isOrphan, err := isFileOrphan(file, referencedFiles, normalizedReferencedFiles, cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to determine if file %s is orphan: %w", file, err)
+ }
+
+ if isOrphan {
+ orphans = append(orphans, file)
+ }
+ }
+
+ return orphans, nil
+}
+
+func isFileOrphan(file string, referencedFiles map[string]bool, normalizedReferencedFiles map[string]string, cfg *orphanCleanupConfig) (bool, error) {
+ normalizedFile := normalizeFilePath(file, cfg)
+
+ if referencedFiles[file] || referencedFiles[normalizedFile] {
+ return false, nil
+ }
+
+ if originalPath, exists := normalizedReferencedFiles[normalizedFile]; exists {
+ err := checkPrefixMismatch(originalPath, file, cfg)
+ if err != nil {
+ return false, err
+ }
+
+ return false, nil
+ }
+
+ return true, nil
+}
+
+func deleteFiles(fs iceio.IO, orphanFiles []string, cfg *orphanCleanupConfig) ([]string, error) {
+ if len(orphanFiles) == 0 {
+ return nil, nil
+ }
+
+ if cfg.maxConcurrency == 1 {
+ return deleteFilesSequential(fs, orphanFiles, cfg)
+ }
+
+ return deleteFilesParallel(fs, orphanFiles, cfg)
+}
+
+func deleteFilesSequential(fs iceio.IO, orphanFiles []string, cfg *orphanCleanupConfig) ([]string, error) {
+ var deletedFiles []string
+
+ deleteFunc := fs.Remove
+ if cfg.deleteFunc != nil {
+ deleteFunc = cfg.deleteFunc
+ }
+
+ var result error
+ for _, file := range orphanFiles {
+ if err := deleteFunc(file); err != nil {
+ result = errors.Join(result, fmt.Errorf("failed to delete orphan file %s: %w", file, err))
+
+ continue
+ }
+ deletedFiles = append(deletedFiles, file)
+ }
+
+ return deletedFiles, result
+}
+
+func deleteFilesParallel(fs iceio.IO, orphanFiles []string, cfg *orphanCleanupConfig) ([]string, error) {
+ deleteFunc := fs.Remove
+ if cfg.deleteFunc != nil {
+ deleteFunc = cfg.deleteFunc
+ }
+
+ in := make(chan string, cfg.maxConcurrency)
+ out := make(chan string, cfg.maxConcurrency)
+ errList := make([][]error, cfg.maxConcurrency)
+
+ go func() {
+ defer close(in)
+ for _, file := range orphanFiles {
+ in <- file
+ }
+ }()
+
+ var wg sync.WaitGroup
+ wg.Add(cfg.maxConcurrency)
+ for i := 0; i < cfg.maxConcurrency; i++ {
+ go func(workerID int) {
+ defer wg.Done()
+ for file := range in {
+ if err := deleteFunc(file); err != nil {
+ errList[workerID] = append(errList[workerID], fmt.Errorf("failed to delete orphan file %s: %w", file, err))
+ } else {
+ out <- file
+ }
+ }
+ }(i)
+ }
+
+ go func() {
+ wg.Wait()
+ close(out)
+ }()
+
+ deletedFiles := make([]string, 0, len(orphanFiles))
+ for file := range out {
+ deletedFiles = append(deletedFiles, file)
+ }
+
+ var allErrors []error
+ for _, workerErrors := range errList {
+ allErrors = append(allErrors, workerErrors...)
+ }
+ err := errors.Join(allErrors...)
+
+ return deletedFiles, err
+}
+
+// normalizeFilePath normalizes file paths for comparison by handling different
+// path representations that might refer to the same file, with support for
+// scheme/authority equivalence as specified in the configuration.
+//
+// This implementation is based on Apache Iceberg's Java DeleteOrphanFiles action:
+// https://github.com/apache/iceberg/blob/07c088fce9c54369864dcb6da16006e78206048b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java#L1
+//
+// The normalization logic specifically follows the ToFileURI.toFileURI() method (lines 542-548):
+// - Line 545: scheme = equalSchemes.getOrDefault(uri.getScheme(), uri.getScheme())
+// - Line 546: authority = equalAuthorities.getOrDefault(uri.getAuthority(), uri.getAuthority())
+//
+// See also: https://iceberg.apache.org/docs/latest/maintenance/#remove-orphan-files
+//
+// Path normalization is essential for orphan cleanup because:
+// 1. Files may be referenced using different but equivalent URI schemes (e.g., s3:// vs s3a:// vs s3n://)
+// 2. Different authorities/endpoints may refer to the same storage (e.g., region-specific S3 endpoints)
+// 3. Path separators and casing may vary across different systems and configurations
+// 4. Without normalization, semantically identical paths would be treated as different,
+// leading to false positives in orphan detection
+func normalizeFilePath(path string, cfg *orphanCleanupConfig) string {
+ // Handle URL-based paths (s3://, gs://, etc.)
+ if strings.Contains(path, "://") {
+ return normalizeURLPath(path, cfg)
+ } else {
+ return normalizeNonURLPath(path)
+ }
+}
+
+// normalizeURLPath normalizes URL-based file paths with scheme/authority equivalence.
+//
+// This function handles the complexities of cloud storage URIs where the same file
+// can be referenced using different but semantically equivalent schemes and authorities.
+//
+// Examples of equivalent schemes (configured via equalSchemes):
+// - s3://bucket/path, s3a://bucket/path, s3n://bucket/path (all refer to S3)
+// - abfs://container@account.dfs.core.windows.net/path, abfss://container@account.dfs.core.windows.net/path (Azure)
+//
+// Examples of equivalent authorities (configured via equalAuthorities):
+// - s3://mybucket.s3.us-west-2.amazonaws.com/path vs s3://s3.us-west-2.amazonaws.com/mybucket/path
+// - Different regional endpoints that serve the same data
+//
+// Based on Apache Iceberg Java's DeleteOrphanFilesSparkAction.toFileURI() normalization (lines 542-548).
+// https://github.com/apache/iceberg/blob/07c088fce9c54369864dcb6da16006e78206048b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java#L1
+func normalizeURLPath(path string, cfg *orphanCleanupConfig) string {
+ parsedURL, err := url.Parse(path)
+ if err != nil {
+ return normalizeNonURLPath(path)
+ }
+
+ normalizedScheme := applySchemeEquivalence(parsedURL.Scheme, cfg.equalSchemes)
+ normalizedAuthority := applyAuthorityEquivalence(parsedURL.Host, cfg.equalAuthorities)
+ normalizedURL := &url.URL{
+ Scheme: normalizedScheme,
+ Host: normalizedAuthority,
+ Path: filepath.Clean(parsedURL.Path),
+ }
+
+ return normalizedURL.String()
+}
+
+// normalizeNonURLPath provides basic path normalization for non-URL paths.
+//
+// Handles file system paths by:
+// 1. Applying filepath.Clean() to resolve "..", ".", and redundant separators
+// 2. Converting Windows-style backslashes to forward slashes for consistency
+//
+// This ensures that paths like "dir/./file", "dir//file", and "dir\file" (on Windows)
+// all normalize to "dir/file" for consistent comparison.
+//
+// Uses filepath.ToSlash() equivalent logic to match Go's standard library approach.
+func normalizeNonURLPath(path string) string {
+ normalized := filepath.Clean(path)
+ // We use this because to handle Windows paths
+ // on all platforms.filepath.ToSlash() only convert the current OS separator, and
+ // we need cross-platform support.
+ return strings.ReplaceAll(normalized, "\\", "/")
+}
+
+// applySchemeEquivalence maps schemes to their equivalent canonical form.
+//
+// Common equivalences include:
+// - S3: s3://, s3a://, s3n:// → s3://
+// - Azure: abfs://, abfss:// → abfs://
+// - HDFS: hdfs://, hdfs+webhdfs:// → hdfs://
+//
+// Based on Apache Iceberg Java's flattenMap() (lines 392-403) and EQUAL_SCHEMES_DEFAULT (line 102).
+// https://github.com/apache/iceberg/blob/07c088fce9c54369864dcb6da16006e78206048b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java#L1
+func applySchemeEquivalence(scheme string, equalSchemes map[string]string) string {
+ if equalSchemes == nil {
+ return scheme
+ }
+ if canonical, exists := equalSchemes[scheme]; exists {
+ return canonical
+ }
+
+ // Check comma-separated lists (e.g., "s3,s3a,s3n" -> "s3")
+ for schemes, canonical := range equalSchemes {
+ if strings.Contains(schemes, ",") {
+ for _, s := range strings.Split(schemes, ",") {
+ if strings.TrimSpace(s) == scheme {
+ return canonical
+ }
+ }
+ }
+ }
+
+ return scheme
+}
+
+// applyAuthorityEquivalence maps authorities to their equivalent canonical form.
+//
+// Different cloud storage endpoints and authorities may serve the same logical storage,
+// but appear different in URIs. This function normalizes these equivalent authorities
+// to enable proper file matching during orphan cleanup.
+//
+// Common authority equivalences include:
+// - Regional S3 endpoints: s3.us-west-2.amazonaws.com, s3-us-west-2.amazonaws.com
+// - S3 path vs virtual-hosted style: bucket.s3.amazonaws.com vs s3.amazonaws.com/bucket
+// - Azure storage endpoints: account.dfs.core.windows.net, account.blob.core.windows.net
+// - Custom endpoints: minio.company.com, s3.company.local
+//
+// Based on Apache Iceberg Java's equalAuthorities logic (lines 546, 161-165, 392-403).
+// https://github.com/apache/iceberg/blob/07c088fce9c54369864dcb6da16006e78206048b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java#L1
+func applyAuthorityEquivalence(authority string, equalAuthorities map[string]string) string {
+ if equalAuthorities == nil {
+ return authority
+ }
+
+ if canonical, exists := equalAuthorities[authority]; exists {
+ return canonical
+ }
+
+ for authorities, canonical := range equalAuthorities {
+ if strings.Contains(authorities, ",") {
+ for _, a := range strings.Split(authorities, ",") {
+ if strings.TrimSpace(a) == authority {
+ return canonical
+ }
+ }
+ }
+ }
+
+ return authority
+}
+
+// checkPrefixMismatch detects and handles prefix mismatches between referenced files and filesystem files
+func checkPrefixMismatch(referencedPath, filesystemPath string, cfg *orphanCleanupConfig) error {
+ // Parse both paths as URLs to compare schemes and authorities
+ refURL, refErr := url.Parse(referencedPath)
+ fsURL, fsErr := url.Parse(filesystemPath)
+
+ if refErr != nil || fsErr != nil {
+ return nil
+ }
+
+ refScheme := applySchemeEquivalence(refURL.Scheme, cfg.equalSchemes)
+ fsScheme := applySchemeEquivalence(fsURL.Scheme, cfg.equalSchemes)
+ refAuth := applyAuthorityEquivalence(refURL.Host, cfg.equalAuthorities)
+ fsAuth := applyAuthorityEquivalence(fsURL.Host, cfg.equalAuthorities)
+
+ // Check for mismatches
+ schemeMismatch := refScheme != fsScheme
+ authMismatch := refAuth != fsAuth
+
+ if !schemeMismatch && !authMismatch {
+ return nil // No mismatch
+ }
+
+ switch cfg.prefixMismatchMode {
+ case PrefixMismatchError:
+ return fmt.Errorf("prefix mismatch detected: referenced=%s (scheme=%s, auth=%s) vs filesystem=%s (scheme=%s, auth=%s)",
+ referencedPath, refScheme, refAuth, filesystemPath, fsScheme, fsAuth)
+ case PrefixMismatchIgnore:
+ return nil // Silently ignore mismatch
+ case PrefixMismatchDelete:
+ return nil // Allow deletion (treat as orphan)
+ default:
+ return fmt.Errorf("unknown prefix mismatch mode: %d", cfg.prefixMismatchMode)
+ }
+}
diff --git a/table/orphan_cleanup_integration_test.go b/table/orphan_cleanup_integration_test.go
new file mode 100644
index 0000000..fef31c9
--- /dev/null
+++ b/table/orphan_cleanup_integration_test.go
@@ -0,0 +1,432 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build integration
+
+package table_test
+
+import (
+ "context"
+ "net/url"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet/pqarrow"
+ "github.com/apache/iceberg-go"
+ "github.com/apache/iceberg-go/catalog"
+ "github.com/apache/iceberg-go/catalog/rest"
+ "github.com/apache/iceberg-go/io"
+ "github.com/apache/iceberg-go/table"
+ "github.com/stretchr/testify/suite"
+)
+
+type OrphanCleanupIntegrationSuite struct {
+ suite.Suite
+
+ ctx context.Context
+ cat *rest.Catalog
+ location string
+}
+
+const (
+ TestNamespace = "orphan_test_ns"
+ TestTableName = "orphan_test_table"
+ TestTableLocation = "s3://warehouse/iceberg/orphan_test"
+ OrphanFilePrefix = "orphan_"
+)
+
+var (
+ tableSchemaSimple = iceberg.NewSchemaWithIdentifiers(1,
+ []int{1},
+ iceberg.NestedField{
+ ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+ iceberg.NestedField{
+ ID: 2, Name: "name", Type: iceberg.PrimitiveTypes.String, Required: true},
+ iceberg.NestedField{
+ ID: 3, Name: "value", Type: iceberg.PrimitiveTypes.Float64, Required: false},
+ )
+)
+
+func (s *OrphanCleanupIntegrationSuite) loadCatalog() *rest.Catalog {
+ cat, err := catalog.Load(s.ctx, "local", iceberg.Properties{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ io.S3Region: "us-east-1",
+ io.S3AccessKeyID: "admin",
+ io.S3SecretAccessKey: "password",
+ })
+ s.Require().NoError(err)
+ s.Require().IsType(&rest.Catalog{}, cat)
+ return cat.(*rest.Catalog)
+}
+
+func (s *OrphanCleanupIntegrationSuite) SetupSuite() {
+ s.ctx = context.Background()
+ s.cat = s.loadCatalog()
+ s.location = TestTableLocation
+
+ // Ensure namespace exists
+ s.ensureNamespace()
+}
+
+func (s *OrphanCleanupIntegrationSuite) TearDownSuite() {
+ if exists, err := s.cat.CheckTableExists(s.ctx, catalog.ToIdentifier(TestNamespace, TestTableName)); err == nil && exists {
+ s.NoError(s.cat.DropTable(s.ctx, catalog.ToIdentifier(TestNamespace, TestTableName)))
+ }
+
+ if exists, err := s.cat.CheckNamespaceExists(s.ctx, catalog.ToIdentifier(TestNamespace)); err == nil && exists {
+ s.NoError(s.cat.DropNamespace(s.ctx, catalog.ToIdentifier(TestNamespace)))
+ }
+}
+
+func (s *OrphanCleanupIntegrationSuite) SetupTest() {
+ if exists, err := s.cat.CheckTableExists(s.ctx, catalog.ToIdentifier(TestNamespace, TestTableName)); err == nil && exists {
+ s.Require().NoError(s.cat.DropTable(s.ctx, catalog.ToIdentifier(TestNamespace, TestTableName)))
+ }
+}
+
+func (s *OrphanCleanupIntegrationSuite) ensureNamespace() {
+ namespace := catalog.ToIdentifier(TestNamespace)
+ exists, err := s.cat.CheckNamespaceExists(s.ctx, namespace)
+ s.Require().NoError(err)
+
+ if exists {
+ for tbl, err := range s.cat.ListTables(s.ctx, namespace) {
+ if err != nil {
+ break
+ }
+ err := s.cat.DropTable(s.ctx, tbl)
+ s.Require().NoError(err)
+ }
+
+ s.Require().NoError(s.cat.DropNamespace(s.ctx, namespace))
+ }
+
+ s.NoError(s.cat.CreateNamespace(s.ctx, namespace,
+ iceberg.Properties{"description": "Namespace for orphan cleanup integration tests"}))
+}
+
+func (s *OrphanCleanupIntegrationSuite) createTableWithData() *table.Table {
+ tbl, err := s.cat.CreateTable(s.ctx, catalog.ToIdentifier(TestNamespace, TestTableName), tableSchemaSimple,
+ catalog.WithLocation(s.location))
+ s.Require().NoError(err)
+ s.Require().NotNil(tbl)
+
+ // Create an arrow table with test data
+ arrSchema, err := table.SchemaToArrowSchema(tableSchemaSimple, nil, false, false)
+ s.Require().NoError(err)
+
+ arrowTable, err := array.TableFromJSON(memory.DefaultAllocator, arrSchema,
+ []string{`[
+ {"id": 1, "name": "alice", "value": 100.5},
+ {"id": 2, "name": "bob", "value": 200.7},
+ {"id": 3, "name": "charlie", "value": 300.9}
+ ]`})
+ s.Require().NoError(err)
+ defer arrowTable.Release()
+
+ dataFile, err := url.JoinPath(s.location, "data", "test_data.parquet")
+ s.Require().NoError(err)
+
+ fs := s.mustFS(tbl)
+ fw, err := fs.(io.WriteFileIO).Create(dataFile)
+ s.Require().NoError(err)
+ s.Require().NoError(pqarrow.WriteTable(arrowTable, fw, arrowTable.NumRows(),
+ nil, pqarrow.DefaultWriterProps()))
+
+ // Commit the data file to the table
+ txn := tbl.NewTransaction()
+ s.Require().NoError(txn.AddFiles(s.ctx, []string{dataFile}, nil, false))
+ updatedTable, err := txn.Commit(s.ctx)
+ s.Require().NoError(err)
+
+ return updatedTable
+}
+
+func (s *OrphanCleanupIntegrationSuite) createOrphanFiles(tbl *table.Table, location string, fileNames []string) []string {
+ fs := s.mustFS(tbl)
+ var createdPaths []string
+
+ for _, fileName := range fileNames {
+ // Create orphan files in the data directory
+ orphanPath, err := url.JoinPath(location, "data", OrphanFilePrefix+fileName)
+ s.Require().NoError(err)
+
+ // Write some fake contents to the orphan file
+ fw, err := fs.(io.WriteFileIO).Create(orphanPath)
+ s.Require().NoError(err)
+
+ content := []byte("This is an orphan file: " + fileName)
+ _, err = fw.Write(content)
+ s.Require().NoError(err)
+ s.Require().NoError(fw.Close())
+
+ createdPaths = append(createdPaths, orphanPath)
+ }
+
+ return createdPaths
+}
+
+func (s *OrphanCleanupIntegrationSuite) mustFS(tbl *table.Table) io.IO {
+ fs, err := tbl.FS(s.ctx)
+ s.Require().NoError(err)
+ return fs
+}
+
+func (s *OrphanCleanupIntegrationSuite) fileExists(fs io.IO, path string) bool {
+ f, err := fs.Open(path)
+ if err != nil {
+ return false
+ }
+ err = f.Close()
+ s.Require().NoError(err)
+ return true
+}
+
+func (s *OrphanCleanupIntegrationSuite) TestOrphanCleanupDryRun() {
+ // Create a table with real data
+ tbl := s.createTableWithData()
+
+ // Debug: Print locations
+ s.T().Logf("Table location: %s", tbl.Location())
+ s.T().Logf("Test location: %s", s.location)
+
+ orphanFiles := s.createOrphanFiles(tbl, s.location, []string{"old1.parquet", "old2.parquet", "old3.parquet"})
+
+ fs := s.mustFS(tbl)
+ for _, orphanFile := range orphanFiles {
+ s.True(s.fileExists(fs, orphanFile), "Orphan file should exist: %s", orphanFile)
+ }
+
+ // Run dry-run orphan cleanup (scan table root where both data and metadata exist)
+ s.T().Logf("Scanning location: %s", tbl.Location())
+ result, err := tbl.DeleteOrphanFiles(s.ctx,
+ table.WithDryRun(true),
+ table.WithLocation(tbl.Location()), // Scan the table root location
+ table.WithFilesOlderThan(-1*time.Hour), // Consider all files (including future timestamp)
+ )
+
+ s.Require().NoError(err)
+ s.Require().NotNil(result)
+
+ s.GreaterOrEqual(len(result.OrphanFileLocations), 3, "Should identify at least 3 orphan files")
+ s.Empty(result.DeletedFiles, "Dry run should not delete any files")
+
+ for _, orphanFile := range orphanFiles {
+ s.True(s.fileExists(fs, orphanFile), "Orphan file should still exist after dry run: %s", orphanFile)
+ }
+
+ for _, orphanFile := range orphanFiles {
+ found := false
+ for _, identified := range result.OrphanFileLocations {
+ if strings.Contains(identified, filepath.Base(orphanFile)) {
+ found = true
+ break
+ }
+ }
+ s.True(found, "Created orphan file should be identified: %s", orphanFile)
+ }
+
+ for _, orphanFile := range orphanFiles {
+ s.NoError(fs.Remove(orphanFile))
+ }
+}
+
+func (s *OrphanCleanupIntegrationSuite) TestOrphanCleanupActualDeletion() {
+ tbl := s.createTableWithData()
+
+ // Create some orphan files that are old enough to be deleted
+ orphanFiles := s.createOrphanFiles(tbl, s.location, []string{"delete1.parquet", "delete2.parquet"})
+
+ // Wait a moment to ensure files are old enough
+ time.Sleep(100 * time.Millisecond)
+
+ // Verify orphan files exist
+ fs := s.mustFS(tbl)
+ for _, orphanFile := range orphanFiles {
+ s.True(s.fileExists(fs, orphanFile), "Orphan file should exist: %s", orphanFile)
+ }
+
+ // Run actual orphan cleanup
+ result, err := tbl.DeleteOrphanFiles(s.ctx,
+ table.WithDryRun(false),
+ table.WithFilesOlderThan(-1*time.Hour), // Consider all files
+ )
+
+ s.Require().NoError(err)
+ s.Require().NotNil(result)
+
+ s.GreaterOrEqual(len(result.OrphanFileLocations), 2, "Should identify at least 2 orphan files")
+ s.GreaterOrEqual(len(result.DeletedFiles), 2, "Should delete at least 2 orphan files")
+
+ for _, orphanFile := range orphanFiles {
+ s.False(s.fileExists(fs, orphanFile), "Orphan file should be deleted: %s", orphanFile)
+ }
+
+ for manifest, err := range tbl.AllManifests(s.ctx) {
+ s.Require().NoError(err)
+ entries, err := manifest.FetchEntries(fs, false)
+ s.Require().NoError(err)
+
+ for _, entry := range entries {
+ dataFile := entry.DataFile().FilePath()
+ s.True(s.fileExists(fs, dataFile), "Data file should still exist: %s", dataFile)
+ }
+ }
+}
+
+func (s *OrphanCleanupIntegrationSuite) TestOrphanCleanupCustomLocation() {
+ tbl := s.createTableWithData()
+
+ customLocation, err := url.JoinPath(s.location, "custom_data")
+ s.Require().NoError(err)
+ orphanFiles := s.createOrphanFiles(tbl, customLocation, []string{"custom1.parquet", "custom2.parquet"})
+
+ fs := s.mustFS(tbl)
+ for _, orphanFile := range orphanFiles {
+ s.True(s.fileExists(fs, orphanFile), "Orphan file should exist: %s", orphanFile)
+ }
+
+ // Wait to ensure files are old enough
+ time.Sleep(100 * time.Millisecond)
+
+ // Run orphan cleanup on table root (which includes the custom subdirectory)
+ result, err := tbl.DeleteOrphanFiles(s.ctx,
+ table.WithLocation(tbl.Location()), // Scan table root to find all subdirectories
+ table.WithFilesOlderThan(-1*time.Hour),
+ table.WithDryRun(false),
+ )
+
+ s.Require().NoError(err)
+ s.Require().NotNil(result)
+
+ s.GreaterOrEqual(len(result.DeletedFiles), 2, "Should delete at least 2 files from custom location")
+
+ for _, orphanFile := range orphanFiles {
+ s.False(s.fileExists(fs, orphanFile), "Custom location orphan file should be deleted: %s", orphanFile)
+ }
+}
+
+func (s *OrphanCleanupIntegrationSuite) TestOrphanCleanupWithConcurrency() {
+ // Create table with real data
+ tbl := s.createTableWithData()
+
+ // Create multiple orphan files for concurrent deletion testing
+ orphanFiles := s.createOrphanFiles(tbl, s.location, []string{
+ "concurrent1.parquet", "concurrent2.parquet", "concurrent3.parquet",
+ "concurrent4.parquet", "concurrent5.parquet",
+ })
+
+ // Wait to ensure files are old enough
+ time.Sleep(100 * time.Millisecond)
+
+ // Test different concurrency levels
+ testCases := []struct {
+ name string
+ concurrency int
+ }{
+ {"sequential", 1},
+ {"parallel_2", 2},
+ {"parallel_4", 4},
+ }
+
+ for _, tc := range testCases {
+ s.Run(tc.name, func() {
+ // Create fresh orphan files for this test case
+ freshOrphanFiles := s.createOrphanFiles(tbl, s.location, []string{
+ tc.name + "_1.parquet", tc.name + "_2.parquet", tc.name + "_3.parquet",
+ })
+
+ time.Sleep(100 * time.Millisecond)
+
+ // Run orphan cleanup with specific concurrency
+ result, err := tbl.DeleteOrphanFiles(s.ctx,
+ table.WithMaxConcurrency(tc.concurrency),
+ table.WithFilesOlderThan(-1*time.Hour),
+ table.WithDryRun(false),
+ )
+
+ s.Require().NoError(err)
+ s.Require().NotNil(result)
+
+ // Verify files were deleted regardless of concurrency level
+ s.GreaterOrEqual(len(result.DeletedFiles), 3, "Should delete at least 3 files with concurrency %d", tc.concurrency)
+
+ fs := s.mustFS(tbl)
+ for _, orphanFile := range freshOrphanFiles {
+ s.False(s.fileExists(fs, orphanFile), "Orphan file should be deleted with concurrency %d: %s", tc.concurrency, orphanFile)
+ }
+ })
+ }
+
+ // Clean up original orphan files
+ fs := s.mustFS(tbl)
+ for _, orphanFile := range orphanFiles {
+ fs.Remove(orphanFile) // Ignore errors, might already be deleted
+ }
+}
+
+func (s *OrphanCleanupIntegrationSuite) TestOrphanCleanupCustomDeleteFunction() {
+ tbl := s.createTableWithData()
+ orphanFiles := s.createOrphanFiles(tbl, s.location, []string{"custom_delete.parquet"})
+
+ // Wait to ensure files are old enough
+ time.Sleep(100 * time.Millisecond)
+
+ var deletedByCustomFunc []string
+ customDeleteFunc := func(filePath string) error {
+ deletedByCustomFunc = append(deletedByCustomFunc, filePath)
+ fs := s.mustFS(tbl)
+ return fs.Remove(filePath)
+ }
+
+ result, err := tbl.DeleteOrphanFiles(s.ctx,
+ table.WithDeleteFunc(customDeleteFunc),
+ table.WithFilesOlderThan(-1*time.Hour),
+ table.WithDryRun(false),
+ table.WithDryRun(false),
+ )
+
+ s.Require().NoError(err)
+ s.Require().NotNil(result)
+
+ s.GreaterOrEqual(len(deletedByCustomFunc), 1, "Custom delete function should be called")
+
+ fs := s.mustFS(tbl)
+ for _, orphanFile := range orphanFiles {
+ s.False(s.fileExists(fs, orphanFile), "Orphan file should be deleted by custom function: %s", orphanFile)
+ }
+
+ for _, orphanFile := range orphanFiles {
+ found := false
+ for _, deletedFile := range deletedByCustomFunc {
+ if strings.Contains(deletedFile, filepath.Base(orphanFile)) {
+ found = true
+ break
+ }
+ }
+ s.True(found, "Orphan file should be processed by custom delete function: %s", orphanFile)
+ }
+}
+
+func TestOrphanCleanupIntegration(t *testing.T) {
+ suite.Run(t, new(OrphanCleanupIntegrationSuite))
+}
diff --git a/table/orphan_cleanup_test.go b/table/orphan_cleanup_test.go
new file mode 100644
index 0000000..3d6bb55
--- /dev/null
+++ b/table/orphan_cleanup_test.go
@@ -0,0 +1,470 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPrefixMismatchMode_String(t *testing.T) {
+ tests := []struct {
+ mode PrefixMismatchMode
+ expected string
+ }{
+ {PrefixMismatchError, "ERROR"},
+ {PrefixMismatchIgnore, "IGNORE"},
+ {PrefixMismatchDelete, "DELETE"},
+ {PrefixMismatchMode(999), "UNKNOWN"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.expected, func(t *testing.T) {
+ assert.Equal(t, tt.expected, tt.mode.String())
+ })
+ }
+}
+
+func TestOrphanCleanupOptions(t *testing.T) {
+ cfg := &orphanCleanupConfig{}
+
+ WithLocation("/test/location")(cfg)
+ assert.Equal(t, "/test/location", cfg.location)
+
+ testDuration := 24 * time.Hour
+ WithFilesOlderThan(testDuration)(cfg)
+ assert.Equal(t, testDuration, cfg.olderThan)
+
+ WithDryRun(true)(cfg)
+ assert.True(t, cfg.dryRun)
+
+ deleteFunc := func(string) error { return nil }
+ WithDeleteFunc(deleteFunc)(cfg)
+ assert.NotNil(t, cfg.deleteFunc)
+
+ WithMaxConcurrency(8)(cfg)
+ assert.Equal(t, 8, cfg.maxConcurrency)
+
+ WithPrefixMismatchMode(PrefixMismatchIgnore)(cfg)
+ assert.Equal(t, PrefixMismatchIgnore, cfg.prefixMismatchMode)
+
+ schemes := map[string]string{"s3,s3a,s3n": "s3"}
+ WithEqualSchemes(schemes)(cfg)
+ assert.Equal(t, schemes, cfg.equalSchemes)
+
+ authorities := map[string]string{"host1,host2": "canonical"}
+ WithEqualAuthorities(authorities)(cfg)
+ assert.Equal(t, authorities, cfg.equalAuthorities)
+}
+
+func TestNormalizeFilePath(t *testing.T) {
+ cfg := &orphanCleanupConfig{
+ equalSchemes: map[string]string{"s3,s3a,s3n": "s3"},
+ equalAuthorities: map[string]string{"endpoint1,endpoint2": "canonical"},
+ }
+
+ tests := []struct {
+ name string
+ input string
+ expected string
+ }{
+ {
+ name: "local_path",
+ input: "/local/path/file.txt",
+ expected: "/local/path/file.txt",
+ },
+ {
+ name: "windows_path",
+ input: "C:\\Windows\\path\\file.txt",
+ expected: "C:/Windows/path/file.txt",
+ },
+ {
+ name: "s3_url",
+ input: "s3://bucket/path/file.txt",
+ expected: "s3://bucket/path/file.txt",
+ },
+ {
+ name: "s3a_to_s3_scheme_equivalence",
+ input: "s3a://bucket/path/file.txt",
+ expected: "s3://bucket/path/file.txt",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := normalizeFilePath(tt.input, cfg)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestNormalizeNonURLPath(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ expected string
+ }{
+ {
+ name: "unix_path",
+ input: "/path/to/file.txt",
+ expected: "/path/to/file.txt",
+ },
+ {
+ name: "windows_path",
+ input: "C:\\Windows\\path\\file.txt",
+ expected: "C:/Windows/path/file.txt",
+ },
+ {
+ name: "relative_path",
+ input: "./relative/path/../file.txt",
+ expected: "relative/file.txt",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := normalizeNonURLPath(tt.input)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestApplySchemeEquivalence(t *testing.T) {
+ equalSchemes := map[string]string{
+ "s3,s3a,s3n": "s3",
+ "gs": "gs",
+ }
+
+ tests := []struct {
+ name string
+ scheme string
+ expected string
+ }{
+ {
+ name: "s3_unchanged",
+ scheme: "s3",
+ expected: "s3",
+ },
+ {
+ name: "s3a_to_s3",
+ scheme: "s3a",
+ expected: "s3",
+ },
+ {
+ name: "s3n_to_s3",
+ scheme: "s3n",
+ expected: "s3",
+ },
+ {
+ name: "gs_unchanged",
+ scheme: "gs",
+ expected: "gs",
+ },
+ {
+ name: "unknown_scheme",
+ scheme: "unknown",
+ expected: "unknown",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := applySchemeEquivalence(tt.scheme, equalSchemes)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestApplyAuthorityEquivalence(t *testing.T) {
+ equalAuthorities := map[string]string{
+ "host1,host2": "canonical",
+ "single": "single",
+ }
+
+ tests := []struct {
+ name string
+ authority string
+ expected string
+ }{
+ {
+ name: "host1_to_canonical",
+ authority: "host1",
+ expected: "canonical",
+ },
+ {
+ name: "host2_to_canonical",
+ authority: "host2",
+ expected: "canonical",
+ },
+ {
+ name: "single_unchanged",
+ authority: "single",
+ expected: "single",
+ },
+ {
+ name: "unknown_authority",
+ authority: "unknown",
+ expected: "unknown",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := applyAuthorityEquivalence(tt.authority, equalAuthorities)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestCheckPrefixMismatch(t *testing.T) {
+ cfg := &orphanCleanupConfig{
+ prefixMismatchMode: PrefixMismatchError,
+ equalSchemes: map[string]string{"s3,s3a,s3n": "s3"},
+ equalAuthorities: map[string]string{"host1,host2": "canonical"},
+ }
+
+ tests := []struct {
+ name string
+ referencedPath string
+ filesystemPath string
+ mode PrefixMismatchMode
+ expectError bool
+ errorContains string
+ }{
+ {
+ name: "no_mismatch",
+ referencedPath: "s3://bucket/path/file.txt",
+ filesystemPath: "s3://bucket/path/file.txt",
+ mode: PrefixMismatchError,
+ expectError: false,
+ },
+ {
+ name: "scheme_mismatch_error_mode",
+ referencedPath: "s3://bucket/path/file.txt",
+ filesystemPath: "gs://bucket/path/file.txt",
+ mode: PrefixMismatchError,
+ expectError: true,
+ errorContains: "prefix mismatch detected",
+ },
+ {
+ name: "scheme_mismatch_ignore_mode",
+ referencedPath: "s3://bucket/path/file.txt",
+ filesystemPath: "gs://bucket/path/file.txt",
+ mode: PrefixMismatchIgnore,
+ expectError: false,
+ },
+ {
+ name: "scheme_mismatch_delete_mode",
+ referencedPath: "s3://bucket/path/file.txt",
+ filesystemPath: "gs://bucket/path/file.txt",
+ mode: PrefixMismatchDelete,
+ expectError: false,
+ },
+ {
+ name: "equivalent_schemes",
+ referencedPath: "s3://bucket/path/file.txt",
+ filesystemPath: "s3a://bucket/path/file.txt",
+ mode: PrefixMismatchError,
+ expectError: false,
+ },
+ {
+ name: "non_url_paths",
+ referencedPath: "/local/path/file.txt",
+ filesystemPath: "/local/path/file.txt",
+ mode: PrefixMismatchError,
+ expectError: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ testCfg := *cfg
+ testCfg.prefixMismatchMode = tt.mode
+
+ err := checkPrefixMismatch(tt.referencedPath, tt.filesystemPath, &testCfg)
+
+ if tt.expectError {
+ assert.Error(t, err)
+ if tt.errorContains != "" {
+ assert.Contains(t, err.Error(), tt.errorContains)
+ }
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
+
+func TestIsFileOrphan(t *testing.T) {
+ cfg := &orphanCleanupConfig{
+ prefixMismatchMode: PrefixMismatchIgnore,
+ equalSchemes: map[string]string{"s3,s3a,s3n": "s3"},
+ }
+
+ referencedFiles := map[string]bool{
+ "s3://bucket/data/file1.parquet": true,
+ "s3://bucket/metadata/manifest.avro": true,
+ "/local/path/file2.parquet": true,
+ }
+
+ tests := []struct {
+ name string
+ file string
+ expectOrphan bool
+ }{
+ {
+ name: "referenced_file_exact_match",
+ file: "s3://bucket/data/file1.parquet",
+ expectOrphan: false,
+ },
+ {
+ name: "referenced_file_scheme_equivalence",
+ file: "s3a://bucket/data/file1.parquet",
+ expectOrphan: false,
+ },
+ {
+ name: "orphan_file",
+ file: "s3://bucket/data/orphan.parquet",
+ expectOrphan: true,
+ },
+ {
+ name: "local_referenced_file",
+ file: "/local/path/file2.parquet",
+ expectOrphan: false,
+ },
+ {
+ name: "local_orphan_file",
+ file: "/local/path/orphan.parquet",
+ expectOrphan: true,
+ },
+ }
+ normalizedReferencedFiles := make(map[string]string)
+ for refPath := range referencedFiles {
+ normalizedPath := normalizeFilePath(refPath, cfg)
+ normalizedReferencedFiles[normalizedPath] = refPath
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ isOrphan, err := isFileOrphan(tt.file, referencedFiles, normalizedReferencedFiles, cfg)
+ require.NoError(t, err)
+ assert.Equal(t, tt.expectOrphan, isOrphan)
+ })
+ }
+}
+
+func TestIdentifyOrphanFiles(t *testing.T) {
+ cfg := &orphanCleanupConfig{
+ prefixMismatchMode: PrefixMismatchIgnore,
+ }
+
+ allFiles := []string{
+ "s3://bucket/data/file1.parquet",
+ "s3://bucket/data/file2.parquet",
+ "s3://bucket/data/orphan1.parquet",
+ "s3://bucket/data/orphan2.parquet",
+ }
+
+ referencedFiles := map[string]bool{
+ "s3://bucket/data/file1.parquet": true,
+ "s3://bucket/data/file2.parquet": true,
+ }
+
+ orphans, err := identifyOrphanFiles(allFiles, referencedFiles, cfg)
+ require.NoError(t, err)
+
+ expectedOrphans := []string{
+ "s3://bucket/data/orphan1.parquet",
+ "s3://bucket/data/orphan2.parquet",
+ }
+
+ assert.ElementsMatch(t, expectedOrphans, orphans)
+}
+
+func TestNormalizeURLPath(t *testing.T) {
+ cfg := &orphanCleanupConfig{
+ equalSchemes: map[string]string{"s3,s3a,s3n": "s3"},
+ equalAuthorities: map[string]string{"host1,host2": "canonical"},
+ }
+
+ tests := []struct {
+ name string
+ input string
+ expected string
+ }{
+ {
+ name: "simple_s3_url",
+ input: "s3://bucket/path/file.txt",
+ expected: "s3://bucket/path/file.txt",
+ },
+ {
+ name: "s3a_to_s3_conversion",
+ input: "s3a://bucket/path/file.txt",
+ expected: "s3://bucket/path/file.txt",
+ },
+ {
+ name: "authority_equivalence",
+ input: "s3://host1/path/file.txt",
+ expected: "s3://canonical/path/file.txt",
+ },
+ {
+ name: "complex_path_cleaning",
+ input: "s3://bucket/path/../other/./file.txt",
+ expected: "s3://bucket/other/file.txt",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := normalizeURLPath(tt.input, cfg)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestNormalizeURLPath_InvalidURL(t *testing.T) {
+ cfg := &orphanCleanupConfig{}
+
+ result := normalizeURLPath("not-a-valid-url", cfg)
+ expected := normalizeNonURLPath("not-a-valid-url")
+ assert.Equal(t, expected, result)
+}
+
+func TestOrphanCleanup_EdgeCases(t *testing.T) {
+ t.Run("prefix_mismatch_unknown_mode", func(t *testing.T) {
+ cfg := &orphanCleanupConfig{
+ prefixMismatchMode: PrefixMismatchMode(999), // Invalid mode
+ }
+
+ err := checkPrefixMismatch("s3://bucket/file", "gs://bucket/file", cfg)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "unknown prefix mismatch mode")
+ })
+
+ t.Run("empty_scheme_authority_maps", func(t *testing.T) {
+ // Test with nil maps
+ result := applySchemeEquivalence("s3", nil)
+ assert.Equal(t, "s3", result)
+
+ result = applyAuthorityEquivalence("host", nil)
+ assert.Equal(t, "host", result)
+ })
+}