You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@answer.apache.org by ku...@apache.org on 2024/07/19 08:04:58 UTC
(incubator-answer-plugins) branch main updated (56a0332 -> 2a39829)
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
from 56a0332 fix(cdn): fix info
new a860ecd fix(cdn): aliyun oss cdn add sync status check
new ce1678c fix(cdn): aliyun oss cdn add sync status check
new 188444f fix(cdn): aliyun oss cdn add sync status check
new af6b174 fix(cdn): aliyun oss cdn add sync status check
new 3dba163 fix(cdn): aliyun oss cdn add sync status check
new 2a39829 fix(cdn): aws s3 cdn add sync status check
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
cdn-aliyunoss/aliyunoss.go | 101 +++++++++++++++++++++++++++++++++----------
cdn-s3/s3.go | 105 ++++++++++++++++++++++++++++++++++-----------
2 files changed, 160 insertions(+), 46 deletions(-)
(incubator-answer-plugins) 05/06: fix(cdn): aliyun oss cdn add sync status check
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit 3dba163b8dbd59c5c788d6c79d930912ede00cdd
Author: kumfo <ku...@sifou.com>
AuthorDate: Fri Jul 19 15:58:51 2024 +0800
fix(cdn): aliyun oss cdn add sync status check
---
cdn-aliyunoss/aliyunoss.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/cdn-aliyunoss/aliyunoss.go b/cdn-aliyunoss/aliyunoss.go
index 2d7c7a2..219dae5 100644
--- a/cdn-aliyunoss/aliyunoss.go
+++ b/cdn-aliyunoss/aliyunoss.go
@@ -114,6 +114,7 @@ func (c *CDN) scanFiles() {
if staticPath == "" {
err := c.scanEmbedFiles("build")
if err != nil {
+ enable = false
log.Error("failed: scan embed files")
return
}
@@ -124,7 +125,8 @@ func (c *CDN) scanFiles() {
err := c.scanStaticPathFiles(staticPath)
if err != nil {
- log.Info("fialed: scan static path files")
+ enable = false
+ log.Error("fialed: scan static path files")
return
}
enable = true
(incubator-answer-plugins) 01/06: fix(cdn): aliyun oss cdn add sync status check
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit a860ecdbd384d9e80019e64e1c2c85cd2797e90f
Author: kumfo <ku...@sifou.com>
AuthorDate: Fri Jul 19 15:44:03 2024 +0800
fix(cdn): aliyun oss cdn add sync status check
---
cdn-aliyunoss/aliyunoss.go | 84 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 65 insertions(+), 19 deletions(-)
diff --git a/cdn-aliyunoss/aliyunoss.go b/cdn-aliyunoss/aliyunoss.go
index a7ffcf2..150d11c 100644
--- a/cdn-aliyunoss/aliyunoss.go
+++ b/cdn-aliyunoss/aliyunoss.go
@@ -45,6 +45,8 @@ const (
defaultMaxFileSize int64 = 10 * 1024 * 1024
)
+var enable = false
+
type CDN struct {
Config *CDNConfig
}
@@ -99,31 +101,49 @@ func (c *CDN) Info() plugin.Info {
// GetStaticPrefix get static prefix
func (c *CDN) GetStaticPrefix() string {
+ if !enable {
+ return ""
+ }
return c.Config.VisitUrlPrefix + c.Config.ObjectKeyPrefix
}
// scanFiles scan all the static files in the build directory
func (c *CDN) scanFiles() {
if staticPath == "" {
- c.scanEmbedFiles("build")
- log.Info("complete scan embed files")
+ err := c.scanEmbedFiles("build")
+ if err != nil {
+ log.Error("failed: scan embed files")
+ return
+ }
+ log.Info("complete: scan embed files")
+ enable = true
+ }
+ err := c.scanStaticPathFiles(staticPath)
+ if err != nil {
+ log.Info("fialed: scan static path files")
return
}
- c.scanStaticPathFiles(staticPath)
- log.Info("complete scan static path files")
+ enable = true
+ log.Info("complete: scan static path files")
}
// scanStaticPathFiles scan static path files
-func (c *CDN) scanStaticPathFiles(fileName string) {
+func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
+ if len(fileName) == 0 {
+ return
+ }
// scan static path files
entry, err := os.ReadDir(fileName)
if err != nil {
- log.Error("read static dir failed: %v", err)
+ log.Error("read static dir failed: ", err, fileName)
return
}
for _, info := range entry {
if info.IsDir() {
- c.scanStaticPathFiles(filepath.Join(fileName, info.Name()))
+ err = c.scanStaticPathFiles(filepath.Join(fileName, info.Name()))
+ if err != nil {
+ return
+ }
continue
}
@@ -133,7 +153,7 @@ func (c *CDN) scanStaticPathFiles(fileName string) {
file, err := os.Open(filePath)
if err != nil {
log.Error("open file failed: %v", err)
- continue
+ return err
}
suffix := staticPath[:1]
@@ -145,34 +165,49 @@ func (c *CDN) scanStaticPathFiles(fileName string) {
// rebuild custom io.Reader
ns := strings.Split(info.Name(), ".")
if info.Name() == "asset-manifest.json" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"/static": "",
}), size)
+ if err != nil {
+ return err
+ }
continue
}
if ns[0] == "main" {
ext := strings.ToLower(filepath.Ext(filePath))
if ext == ".js" || ext == ".map" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"static": "",
"=\"/\",": "=\"\",",
}), size)
+
+ if err != nil {
+ return err
+ }
continue
}
if ext == ".css" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"url(/static": "url(../../static",
}), size)
+
+ if err != nil {
+ return err
+ }
continue
}
}
- c.Upload(filePath, file, size)
+ err = c.Upload(filePath, file, size)
+ if err != nil {
+ return err
+ }
}
+ return nil
}
-func (c *CDN) scanEmbedFiles(fileName string) {
+func (c *CDN) scanEmbedFiles(fileName string) (err error) {
entry, err := ui.Build.ReadDir(fileName)
if err != nil {
log.Error("read static dir failed: %v", err)
@@ -180,7 +215,7 @@ func (c *CDN) scanEmbedFiles(fileName string) {
}
for _, info := range entry {
if info.IsDir() {
- c.scanEmbedFiles(filepath.Join(fileName, info.Name()))
+ err = c.scanEmbedFiles(filepath.Join(fileName, info.Name()))
continue
}
@@ -191,7 +226,7 @@ func (c *CDN) scanEmbedFiles(fileName string) {
defer file.Close()
if err != nil {
log.Error("open file failed: %v", err)
- continue
+ return err
}
filePath = strings.TrimPrefix(filePath, "build/")
@@ -199,31 +234,41 @@ func (c *CDN) scanEmbedFiles(fileName string) {
// rebuild custom io.Reader
ns := strings.Split(info.Name(), ".")
if info.Name() == "asset-manifest.json" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"/static": "",
}), size)
+ if err != nil {
+ return err
+ }
continue
}
if ns[0] == "main" {
ext := strings.ToLower(filepath.Ext(filePath))
if ext == ".js" || ext == ".map" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"static": "",
"=\"/\",": "=\"\",",
}), size)
+ if err != nil {
+ return err
+ }
continue
}
if ext == ".css" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"url(/static": "url(../../static",
}), size)
+ if err != nil {
+ return err
+ }
continue
}
}
c.Upload(filePath, file, size)
}
+ return nil
}
func (c *CDN) rebuildReader(file io.Reader, replaceMap map[string]string) io.Reader {
@@ -254,7 +299,7 @@ func (c *CDN) rebuildReader(file io.Reader, replaceMap map[string]string) io.Rea
return strings.NewReader(res)
}
-func (c *CDN) Upload(filePath string, file io.Reader, size int64) {
+func (c *CDN) Upload(filePath string, file io.Reader, size int64) (err error) {
client, err := oss.New(c.Config.Endpoint, c.Config.AccessKeyID, c.Config.AccessKeySecret)
if err != nil {
log.Error(plugin.MakeTranslator(i18n.ErrMisStorageConfig), err)
@@ -287,6 +332,7 @@ func (c *CDN) Upload(filePath string, file io.Reader, size int64) {
return
}
defer respBody.Close()
+ return nil
}
func (c *CDN) createObjectKey(filePath string) string {
(incubator-answer-plugins) 06/06: fix(cdn): aws s3 cdn add sync status check
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit 2a39829360936b9ec9ff8550a4aa999d743ed084
Author: kumfo <ku...@sifou.com>
AuthorDate: Fri Jul 19 16:04:15 2024 +0800
fix(cdn): aws s3 cdn add sync status check
---
cdn-s3/s3.go | 105 +++++++++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 81 insertions(+), 24 deletions(-)
diff --git a/cdn-s3/s3.go b/cdn-s3/s3.go
index 29ab361..a75bf8a 100644
--- a/cdn-s3/s3.go
+++ b/cdn-s3/s3.go
@@ -30,6 +30,7 @@ import (
"github.com/apache/incubator-answer/ui"
"github.com/segmentfault/pacman/log"
"io"
+ "io/fs"
"os"
"path/filepath"
"strconv"
@@ -39,7 +40,10 @@ import (
"github.com/apache/incubator-answer/plugin"
)
-var staticPath = os.Getenv("ANSWER_STATIC_PATH")
+var (
+ staticPath = os.Getenv("ANSWER_STATIC_PATH")
+ enable = false
+)
//go:embed info.yaml
var Info embed.FS
@@ -88,22 +92,42 @@ func (c *CDN) Info() plugin.Info {
// GetStaticPrefix get static prefix
func (c *CDN) GetStaticPrefix() string {
+ if !enable {
+ return ""
+ }
return c.Config.VisitUrlPrefix + c.Config.ObjectKeyPrefix
}
// scanFiles scan all the static files in the build directory
func (c *CDN) scanFiles() {
if staticPath == "" {
- c.scanEmbedFiles("build")
- log.Info("complete scan embed files")
+ err := c.scanEmbedFiles("build")
+ if err != nil {
+ enable = false
+ log.Error("failed: scan embed files")
+ return
+ }
+ log.Info("complete: scan embed files")
+ enable = true
+ return
+ }
+
+ err := c.scanStaticPathFiles(staticPath)
+ if err != nil {
+ enable = false
+ log.Error("fialed: scan static path files")
return
}
- c.scanStaticPathFiles(staticPath)
- log.Info("complete scan static path files")
+ enable = true
+ log.Info("complete: scan static path files")
}
// scanStaticPathFiles scan static path files
-func (c *CDN) scanStaticPathFiles(fileName string) {
+func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
+ if len(fileName) == 0 {
+ return
+ }
+
// scan static path files
entry, err := os.ReadDir(fileName)
if err != nil {
@@ -112,17 +136,20 @@ func (c *CDN) scanStaticPathFiles(fileName string) {
}
for _, info := range entry {
if info.IsDir() {
- c.scanStaticPathFiles(filepath.Join(fileName, info.Name()))
- continue
+ err = c.scanStaticPathFiles(filepath.Join(fileName, info.Name()))
+ if err != nil {
+ return
+ }
}
+ var file *os.File
filePath := filepath.Join(fileName, info.Name())
fi, _ := info.Info()
size := fi.Size()
- file, err := os.Open(filePath)
+ file, err = os.Open(filePath)
if err != nil {
log.Error("open file failed: %v", err)
- continue
+ return
}
suffix := staticPath[:1]
@@ -134,34 +161,47 @@ func (c *CDN) scanStaticPathFiles(fileName string) {
// rebuild custom io.Reader
ns := strings.Split(info.Name(), ".")
if info.Name() == "asset-manifest.json" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"/static": "",
}), size)
+ if err != nil {
+ return
+ }
continue
}
if ns[0] == "main" {
ext := strings.ToLower(filepath.Ext(filePath))
if ext == ".js" || ext == ".map" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"static": "",
"=\"/\",": "=\"\",",
}), size)
+ if err != nil {
+ return
+ }
continue
}
if ext == ".css" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"url(/static": "url(../../static",
}), size)
+ if err != nil {
+ return
+ }
continue
}
}
- c.Upload(filePath, file, size)
+ err = c.Upload(filePath, file, size)
+ if err != nil {
+ return
+ }
}
+ return
}
-func (c *CDN) scanEmbedFiles(fileName string) {
+func (c *CDN) scanEmbedFiles(fileName string) (err error) {
entry, err := ui.Build.ReadDir(fileName)
if err != nil {
log.Error("read static dir failed: %v", err)
@@ -169,18 +209,22 @@ func (c *CDN) scanEmbedFiles(fileName string) {
}
for _, info := range entry {
if info.IsDir() {
- c.scanEmbedFiles(filepath.Join(fileName, info.Name()))
+ err = c.scanEmbedFiles(filepath.Join(fileName, info.Name()))
+ if err != nil {
+ return
+ }
continue
}
+ var file fs.File
filePath := filepath.Join(fileName, info.Name())
fi, _ := info.Info()
size := fi.Size()
- file, err := ui.Build.Open(filePath)
+ file, err = ui.Build.Open(filePath)
defer file.Close()
if err != nil {
log.Error("open file failed: %v", err)
- continue
+ return
}
filePath = strings.TrimPrefix(filePath, "build/")
@@ -188,32 +232,45 @@ func (c *CDN) scanEmbedFiles(fileName string) {
// rebuild custom io.Reader
ns := strings.Split(info.Name(), ".")
if info.Name() == "asset-manifest.json" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"/static": "",
}), size)
+ if err != nil {
+ return
+ }
continue
}
if ns[0] == "main" {
ext := strings.ToLower(filepath.Ext(filePath))
if ext == ".js" || ext == ".map" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"\"static": "",
"=\"/\",": "=\"\",",
}), size)
+ if err != nil {
+ return
+ }
continue
}
if ext == ".css" {
- c.Upload(filePath, c.rebuildReader(file, map[string]string{
+ err = c.Upload(filePath, c.rebuildReader(file, map[string]string{
"url(/static": "url(../../static",
}), size)
+ if err != nil {
+ return
+ }
continue
}
}
- c.Upload(filePath, c.rebuildReader(file, nil), size)
+ err = c.Upload(filePath, c.rebuildReader(file, nil), size)
+ if err != nil {
+ return
+ }
}
+ return
}
func (c *CDN) rebuildReader(file io.Reader, replaceMap map[string]string) io.ReadSeeker {
@@ -246,7 +303,7 @@ func (c *CDN) rebuildReader(file io.Reader, replaceMap map[string]string) io.Rea
return strings.NewReader(res)
}
-func (c *CDN) Upload(filePath string, file io.ReadSeeker, size int64) {
+func (c *CDN) Upload(filePath string, file io.ReadSeeker, size int64) (err error) {
if !c.CheckFileType(filePath) {
log.Error(plugin.MakeTranslator(i18n.ErrUnsupportedFileType), filePath)
@@ -260,7 +317,7 @@ func (c *CDN) Upload(filePath string, file io.ReadSeeker, size int64) {
objectKey := c.createObjectKey(filePath)
- err := c.Client.PutObject(objectKey, strings.ToLower(filepath.Ext(filePath)), file)
+ err = c.Client.PutObject(objectKey, strings.ToLower(filepath.Ext(filePath)), file)
if err != nil {
log.Error(plugin.MakeTranslator(i18n.ErrUploadFileFailed), err)
return
(incubator-answer-plugins) 03/06: fix(cdn): aliyun oss cdn add sync status check
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit 188444f56e9ee8fe9ea18443d0663f9ff8bac563
Author: kumfo <ku...@sifou.com>
AuthorDate: Fri Jul 19 15:54:18 2024 +0800
fix(cdn): aliyun oss cdn add sync status check
---
cdn-aliyunoss/aliyunoss.go | 25 ++++++++++++++-----------
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/cdn-aliyunoss/aliyunoss.go b/cdn-aliyunoss/aliyunoss.go
index 9001674..a145341 100644
--- a/cdn-aliyunoss/aliyunoss.go
+++ b/cdn-aliyunoss/aliyunoss.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/incubator-answer/ui"
"github.com/segmentfault/pacman/log"
"io"
+ "io/fs"
"os"
"path/filepath"
"strconv"
@@ -148,13 +149,14 @@ func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
continue
}
+ var file fs.File
filePath := filepath.Join(fileName, info.Name())
fi, _ := info.Info()
size := fi.Size()
- file, err := os.Open(filePath)
+ file, err = os.Open(filePath)
if err != nil {
log.Error("open file failed: %v", err)
- return err
+ return
}
suffix := staticPath[:1]
@@ -170,7 +172,7 @@ func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
"\"/static": "",
}), size)
if err != nil {
- return err
+ return
}
continue
}
@@ -183,7 +185,7 @@ func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
}), size)
if err != nil {
- return err
+ return
}
continue
}
@@ -194,7 +196,7 @@ func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
}), size)
if err != nil {
- return err
+ return
}
continue
}
@@ -202,7 +204,7 @@ func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
err = c.Upload(filePath, file, size)
if err != nil {
- return err
+ return
}
}
return nil
@@ -223,14 +225,15 @@ func (c *CDN) scanEmbedFiles(fileName string) (err error) {
continue
}
+ var file fs.File
filePath := filepath.Join(fileName, info.Name())
fi, _ := info.Info()
size := fi.Size()
- file, err := ui.Build.Open(filePath)
+ file, err = ui.Build.Open(filePath)
defer file.Close()
if err != nil {
log.Error("open file failed: %v", err)
- return err
+ return
}
filePath = strings.TrimPrefix(filePath, "build/")
@@ -242,7 +245,7 @@ func (c *CDN) scanEmbedFiles(fileName string) (err error) {
"\"/static": "",
}), size)
if err != nil {
- return err
+ return
}
continue
}
@@ -254,7 +257,7 @@ func (c *CDN) scanEmbedFiles(fileName string) (err error) {
"=\"/\",": "=\"\",",
}), size)
if err != nil {
- return err
+ return
}
continue
}
@@ -264,7 +267,7 @@ func (c *CDN) scanEmbedFiles(fileName string) (err error) {
"url(/static": "url(../../static",
}), size)
if err != nil {
- return err
+ return
}
continue
}
(incubator-answer-plugins) 02/06: fix(cdn): aliyun oss cdn add sync status check
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit ce1678c74b7575fd43564de941610f3cafe2f6c7
Author: kumfo <ku...@sifou.com>
AuthorDate: Fri Jul 19 15:48:01 2024 +0800
fix(cdn): aliyun oss cdn add sync status check
---
cdn-aliyunoss/aliyunoss.go | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/cdn-aliyunoss/aliyunoss.go b/cdn-aliyunoss/aliyunoss.go
index 150d11c..9001674 100644
--- a/cdn-aliyunoss/aliyunoss.go
+++ b/cdn-aliyunoss/aliyunoss.go
@@ -35,7 +35,10 @@ import (
"strings"
)
-var staticPath = os.Getenv("ANSWER_STATIC_PATH")
+var (
+ staticPath = os.Getenv("ANSWER_STATIC_PATH")
+ enable = false
+)
//go:embed info.yaml
var Info embed.FS
@@ -45,8 +48,6 @@ const (
defaultMaxFileSize int64 = 10 * 1024 * 1024
)
-var enable = false
-
type CDN struct {
Config *CDNConfig
}
@@ -216,6 +217,9 @@ func (c *CDN) scanEmbedFiles(fileName string) (err error) {
for _, info := range entry {
if info.IsDir() {
err = c.scanEmbedFiles(filepath.Join(fileName, info.Name()))
+ if err != nil {
+ return
+ }
continue
}
(incubator-answer-plugins) 04/06: fix(cdn): aliyun oss cdn add sync status check
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kumfo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-answer-plugins.git
commit af6b1745b4a29202c2e6c3c13032e396516be816
Author: kumfo <ku...@sifou.com>
AuthorDate: Fri Jul 19 15:57:36 2024 +0800
fix(cdn): aliyun oss cdn add sync status check
---
cdn-aliyunoss/aliyunoss.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/cdn-aliyunoss/aliyunoss.go b/cdn-aliyunoss/aliyunoss.go
index a145341..2d7c7a2 100644
--- a/cdn-aliyunoss/aliyunoss.go
+++ b/cdn-aliyunoss/aliyunoss.go
@@ -119,7 +119,9 @@ func (c *CDN) scanFiles() {
}
log.Info("complete: scan embed files")
enable = true
+ return
}
+
err := c.scanStaticPathFiles(staticPath)
if err != nil {
log.Info("fialed: scan static path files")
@@ -149,7 +151,7 @@ func (c *CDN) scanStaticPathFiles(fileName string) (err error) {
continue
}
- var file fs.File
+ var file *os.File
filePath := filepath.Join(fileName, info.Name())
fi, _ := info.Info()
size := fi.Size()