[incubator-devlake] branch main updated: [fix-2829]: Compress tar-archived logs with gzip (#2831)

[kl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new de294f56 [fix-2829]: Compress tar-archived logs with gzip (#2831)
de294f56 is described below

commit de294f56963c3f4317d664fddb857a4c632b2ee5
Author: Keon Amini <ke...@merico.dev>
AuthorDate: Thu Aug 25 21:11:37 2022 -0500

    [fix-2829]: Compress tar-archived logs with gzip (#2831)
    
    * fix: change log archiving format from tar to zip
    closes #2829
    
    * feat: supporting gzip compression now
---
 services/pipeline.go |  2 +-
 utils/io.go          | 85 ++++++++++++++++++++++++++++++++++++++++------------
 2 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/services/pipeline.go b/services/pipeline.go
index 820a306b..e4d9612e 100644
--- a/services/pipeline.go
+++ b/services/pipeline.go
@@ -208,7 +208,7 @@ func GetPipelineLogsArchivePath(pipeline *models.Pipeline) (string, error) {
 		return "", err
 	}
 	archive := fmt.Sprintf("%s/%s/logging.tar.gz", os.TempDir(), uuid.New())
-	if err = utils.CreateArchive(archive, true, logPath); err != nil {
+	if err = utils.CreateGZipArchive(archive, fmt.Sprintf("%s/*", logPath)); err != nil {
 		return "", err
 	}
 	return archive, err
diff --git a/utils/io.go b/utils/io.go
index d0c43365..808299d1 100644
--- a/utils/io.go
+++ b/utils/io.go
@@ -18,56 +18,85 @@ limitations under the License.
 package utils
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"github.com/viant/afs"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // fs abstract filesystem interface singleton instance
 var fs = afs.New()
 
-// CreateArchive creates a tar archive and writes the files/directories associated with the `targetPaths` to it.
-// 'relativeCopy' = true will copy the contents inside each sourcePath (directory) over. If the sourcePath is a file, it is directly copied over.
-func CreateArchive(archivePath string, relativeCopy bool, sourcePaths ...string) error {
+// CreateZipArchive creates a zip archive and writes the files/directories associated with the `sourcePaths` to it.
+// If a sourcePath directory ends with /*, then its contents are copied over, but not the directory itself
+func CreateZipArchive(archivePath string, sourcePaths ...string) error {
+	return createArchive("zip", archivePath, sourcePaths...)
+}
+
+// CreateGZipArchive creates a tar archive, compresses it with gzip and writes the files/directories associated with the `sourcePaths` to it.
+// If a sourcePath directory ends with /*, then its contents are copied over, but not the directory itself
+func CreateGZipArchive(archivePath string, sourcePaths ...string) error {
+	err := createArchive("tar", archivePath, sourcePaths...)
+	if err != nil {
+		return err
+	}
+	// now gzip it
+	err = toGzip(archivePath)
+	if err != nil {
+		return fmt.Errorf("error compressing archive to gzip: %w", err)
+	}
+	return nil
+}
+
+func createArchive(archiveType string, archivePath string, sourcePaths ...string) error {
 	for _, sourcePath := range sourcePaths {
+		relativeCopy := false
+		if strings.HasSuffix(sourcePath, "/*") {
+			sourcePath = sourcePath[0 : len(sourcePath)-2]
+			relativeCopy = true
+		}
 		srcPathAbs, err := filepath.Abs(sourcePath)
 		if err != nil {
-			return fmt.Errorf("error getting absolute path of %s: %v", sourcePaths, err)
+			return fmt.Errorf("error getting absolute path of %s: %w", sourcePaths, err)
 		}
 		archivePathAbs, err := filepath.Abs(archivePath)
 		if err != nil {
-			return fmt.Errorf("error getting absolute path of %s: %v", archivePath, err)
+			return fmt.Errorf("error getting absolute path of %s: %w", archivePath, err)
 		}
 		srcInfo, err := os.Stat(srcPathAbs)
 		if err != nil {
-			return fmt.Errorf("error getting stats of path %s: %v", srcPathAbs, err)
+			return fmt.Errorf("error getting stats of path %s: %w", srcPathAbs, err)
 		}
-		if relativeCopy {
-			if srcInfo.IsDir() {
-				err = copyContentsToArchive(srcPathAbs, archivePathAbs)
-			} else {
-				err = copyToArchive(srcPathAbs, archivePathAbs, srcInfo.Name())
-			}
+		if relativeCopy && srcInfo.IsDir() {
+			err = copyContentsToArchive(archiveType, srcPathAbs, archivePathAbs)
 		} else {
-			err = copyToArchive(srcPathAbs, archivePathAbs, sourcePath)
+			if relativeCopy {
+				sourcePath = srcInfo.Name()
+			}
+			// directly copies over the src path as the dest path in the archive (can be improved later if needed to guard against abs paths)
+			err = copyToArchive(archiveType, srcPathAbs, archivePathAbs, sourcePath)
 		}
 		if err != nil {
-			return fmt.Errorf("error trying to copy data to archive: %v", err)
+			return fmt.Errorf("error trying to copy data to archive: %w", err)
 		}
 	}
 	return nil
 }
 
-func copyContentsToArchive(absSourcePath string, absArchivePath string) error {
+func copyContentsToArchive(archiveType string, absSourcePath string, absArchivePath string) error {
 	var files []os.DirEntry
 	files, err := os.ReadDir(absSourcePath)
 	if err != nil {
 		return err
 	}
-	for _, file := range files {
-		err = fs.Copy(context.Background(), fmt.Sprintf("file://%s/%s", absSourcePath, file.Name()), fmt.Sprintf("file:%s/tar:///%s", absArchivePath, file.Name()))
+	for _, desPath := range files {
+		archiveDest := desPath.Name()
+		src := fmt.Sprintf("%s/%s", absSourcePath, archiveDest)
+		err = copyToArchive(archiveType, src, absArchivePath, archiveDest)
 		if err != nil {
 			return err
 		}
@@ -75,6 +104,24 @@ func copyContentsToArchive(absSourcePath string, absArchivePath string) error {
 	return nil
 }
 
-func copyToArchive(absSourcePath string, absArchivePath string, filename string) error {
-	return fs.Copy(context.Background(), fmt.Sprintf("file://%s", absSourcePath), fmt.Sprintf("file:%s/tar:///%s", absArchivePath, filename))
+func copyToArchive(archiveType string, absSourcePath string, absArchivePath string, archiveDest string) error {
+	src := fmt.Sprintf("file://%s", absSourcePath)
+	dst := fmt.Sprintf("file:%s/%s:///%s", absArchivePath, archiveType, archiveDest)
+	return fs.Copy(context.Background(), src, dst)
+}
+
+func toGzip(archivePath string) error {
+	info, _ := os.Stat(archivePath)
+	b, err := os.ReadFile(archivePath)
+	if err != nil {
+		return err
+	}
+	var buf bytes.Buffer
+	w := gzip.NewWriter(&buf)
+	_, err = w.Write(b)
+	_ = w.Close()
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(archivePath, buf.Bytes(), info.Mode().Perm())
 }