You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@answer.apache.org by li...@apache.org on 2025/01/10 02:49:35 UTC
(answer) 01/01: feat(service): implement file cleanup and deletion functionality
This is an automated email from the ASF dual-hosted git repository.
linkinstar pushed a commit to branch feat/1.4.5/file
in repository https://gitbox.apache.org/repos/asf/answer.git
commit 5cffab864939e08bdf351feb9c8fc9b64934cdfa
Author: LinkinStars <li...@foxmail.com>
AuthorDate: Fri Jan 10 10:43:35 2025 +0800
feat(service): implement file cleanup and deletion functionality
---
cmd/wire_gen.go | 12 +-
configs/config.yaml | 3 +
go.mod | 2 +-
internal/base/constant/upload.go | 1 +
internal/base/cron/cron.go | 43 +++++-
internal/controller/upload_controller.go | 9 +-
internal/entity/file_record_entity.go | 45 ++++++
internal/migrations/migrations.go | 1 +
.../service_config.go => migrations/v25.go} | 13 +-
internal/repo/answer/answer_repo.go | 19 ++-
internal/repo/file_record/file_record_repo.go | 84 ++++++++++
internal/repo/provider.go | 2 +
internal/repo/question/question_repo.go | 17 ++
internal/repo/revision/revision_repo.go | 12 +-
internal/router/static_router.go | 12 +-
internal/service/content/question_service.go | 2 +
.../service/file_record/file_record_service.go | 172 +++++++++++++++++++++
internal/service/object_info/object_info.go | 3 +-
internal/service/provider.go | 2 +
internal/service/revision/revision.go | 1 +
internal/service/service_config/service_config.go | 5 +-
internal/service/tag_common/tag_common.go | 9 +-
internal/service/uploader/upload.go | 52 +++++--
pkg/writer/writer.go | 5 +
24 files changed, 483 insertions(+), 43 deletions(-)
diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go
index b44fadfb..702f60df 100644
--- a/cmd/wire_gen.go
+++ b/cmd/wire_gen.go
@@ -48,6 +48,7 @@ import (
"github.com/apache/answer/internal/repo/comment"
"github.com/apache/answer/internal/repo/config"
"github.com/apache/answer/internal/repo/export"
+ "github.com/apache/answer/internal/repo/file_record"
"github.com/apache/answer/internal/repo/limit"
"github.com/apache/answer/internal/repo/meta"
notification2 "github.com/apache/answer/internal/repo/notification"
@@ -84,6 +85,7 @@ import (
"github.com/apache/answer/internal/service/dashboard"
"github.com/apache/answer/internal/service/event_queue"
export2 "github.com/apache/answer/internal/service/export"
+ file_record2 "github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/service/follow"
"github.com/apache/answer/internal/service/importer"
meta2 "github.com/apache/answer/internal/service/meta"
@@ -227,10 +229,10 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
contentRevisionService := content.NewRevisionService(revisionRepo, userCommon, questionCommon, answerService, objService, questionRepo, answerRepo, tagRepo, tagCommonService, notificationQueueService, activityQueueService, reportRepo, reviewService, reviewActivityRepo)
revisionController := controller.NewRevisionController(contentRevisionService, rankService)
rankController := controller.NewRankController(rankService)
- badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)
+ userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)
notificationRepo := notification2.NewNotificationRepo(dataData)
pluginUserConfigRepo := plugin_config.NewPluginUserConfigRepo(dataData)
- userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)
+ badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)
userAdminService := user_admin.NewUserAdminService(userAdminRepo, userRoleRelService, authService, userCommon, userActiveActivityRepo, siteInfoCommonService, emailService, questionRepo, answerRepo, commentCommonRepo, userExternalLoginRepo, notificationRepo, pluginUserConfigRepo, badgeAwardRepo)
userAdminController := controller_admin.NewUserAdminController(userAdminService)
reasonRepo := reason.NewReasonRepo(configService)
@@ -246,7 +248,9 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
notificationController := controller.NewNotificationController(notificationService, rankService)
dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData)
dashboardController := controller.NewDashboardController(dashboardService)
- uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService)
+ fileRecordRepo := file_record.NewFileRecordRepo(dataData)
+ fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService)
+ uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService)
uploadController := controller.NewUploadController(uploaderService)
activityActivityRepo := activity.NewActivityRepo(dataData, configService)
activityCommon := activity_common2.NewActivityCommon(activityRepo, activityQueueService)
@@ -287,7 +291,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
renderController := controller.NewRenderController()
pluginAPIRouter := router.NewPluginAPIRouter(connectorController, userCenterController, captchaController, embedController, renderController)
ginEngine := server.NewHTTPServer(debug, staticRouter, answerAPIRouter, swaggerRouter, uiRouter, authUserMiddleware, avatarMiddleware, shortIDMiddleware, templateRouter, pluginAPIRouter, uiConf)
- scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService)
+ scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService, fileRecordService, serviceConf)
application := newApplication(serverConf, ginEngine, scheduledTaskManager)
return application, func() {
cleanup2()
diff --git a/configs/config.yaml b/configs/config.yaml
index 52671010..69c4e303 100644
--- a/configs/config.yaml
+++ b/configs/config.yaml
@@ -33,6 +33,9 @@ swaggerui:
address: ':80'
service_config:
upload_path: "/data/uploads"
+ clean_up_uploads: true
+ clean_orphan_uploads_period_hours: 48
+ purge_deleted_files_period_days: 30
ui:
public_url: '/'
api_url: '/'
diff --git a/go.mod b/go.mod
index 7e207e6b..89c6876b 100644
--- a/go.mod
+++ b/go.mod
@@ -60,6 +60,7 @@ require (
golang.org/x/crypto v0.27.0
golang.org/x/image v0.20.0
golang.org/x/net v0.29.0
+ golang.org/x/text v0.18.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.33.0
@@ -160,7 +161,6 @@ require (
golang.org/x/arch v0.10.0 // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/sys v0.25.0 // indirect
- golang.org/x/text v0.18.0 // indirect
golang.org/x/tools v0.25.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
diff --git a/internal/base/constant/upload.go b/internal/base/constant/upload.go
index c01c9279..d9001fde 100644
--- a/internal/base/constant/upload.go
+++ b/internal/base/constant/upload.go
@@ -25,4 +25,5 @@ const (
PostSubPath = "post"
BrandingSubPath = "branding"
FilesPostSubPath = "files/post"
+ DeletedSubPath = "deleted"
)
diff --git a/internal/base/cron/cron.go b/internal/base/cron/cron.go
index 2e7a92ab..98e59dec 100644
--- a/internal/base/cron/cron.go
+++ b/internal/base/cron/cron.go
@@ -24,6 +24,8 @@ import (
"fmt"
"github.com/apache/answer/internal/service/content"
+ "github.com/apache/answer/internal/service/file_record"
+ "github.com/apache/answer/internal/service/service_config"
"github.com/apache/answer/internal/service/siteinfo_common"
"github.com/robfig/cron/v3"
"github.com/segmentfault/pacman/log"
@@ -31,29 +33,36 @@ import (
// ScheduledTaskManager scheduled task manager
type ScheduledTaskManager struct {
- siteInfoService siteinfo_common.SiteInfoCommonService
- questionService *content.QuestionService
+ siteInfoService siteinfo_common.SiteInfoCommonService
+ questionService *content.QuestionService
+ fileRecordService *file_record.FileRecordService
+ serviceConfig *service_config.ServiceConfig
}
// NewScheduledTaskManager new scheduled task manager
func NewScheduledTaskManager(
siteInfoService siteinfo_common.SiteInfoCommonService,
questionService *content.QuestionService,
+ fileRecordService *file_record.FileRecordService,
+ serviceConfig *service_config.ServiceConfig,
) *ScheduledTaskManager {
manager := &ScheduledTaskManager{
- siteInfoService: siteInfoService,
- questionService: questionService,
+ siteInfoService: siteInfoService,
+ questionService: questionService,
+ fileRecordService: fileRecordService,
+ serviceConfig: serviceConfig,
}
return manager
}
func (s *ScheduledTaskManager) Run() {
- fmt.Println("start cron")
+ log.Infof("cron job manager start")
+
s.questionService.SitemapCron(context.Background())
c := cron.New()
_, err := c.AddFunc("0 */1 * * *", func() {
ctx := context.Background()
- fmt.Println("sitemap cron execution")
+ log.Infof("sitemap cron execution")
s.questionService.SitemapCron(ctx)
})
if err != nil {
@@ -62,12 +71,32 @@ func (s *ScheduledTaskManager) Run() {
_, err = c.AddFunc("0 */1 * * *", func() {
ctx := context.Background()
- fmt.Println("refresh hottest cron execution")
+ log.Infof("refresh hottest cron execution")
s.questionService.RefreshHottestCron(ctx)
})
if err != nil {
log.Error(err)
}
+ if s.serviceConfig.CleanUpUploads {
+ log.Infof("clean up uploads cron enabled")
+
+ conf := s.serviceConfig
+ _, err = c.AddFunc(fmt.Sprintf("0 */%d * * *", conf.CleanOrphanUploadsPeriodHours), func() {
+ log.Infof("clean orphan upload files cron execution")
+ s.fileRecordService.CleanOrphanUploadFiles(context.Background())
+ })
+ if err != nil {
+ log.Error(err)
+ }
+
+ _, err = c.AddFunc(fmt.Sprintf("0 0 */%d * *", conf.PurgeDeletedFilesPeriodDays), func() {
+ log.Infof("purge deleted files cron execution")
+ s.fileRecordService.PurgeDeletedFiles(context.Background())
+ })
+ if err != nil {
+ log.Error(err)
+ }
+ }
c.Start()
}
diff --git a/internal/controller/upload_controller.go b/internal/controller/upload_controller.go
index 0bd74bf3..337cf83d 100644
--- a/internal/controller/upload_controller.go
+++ b/internal/controller/upload_controller.go
@@ -70,19 +70,20 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) {
)
source := ctx.PostForm("source")
+ userID := middleware.GetLoginUserIDFromContext(ctx)
switch source {
case fileFromAvatar:
- url, err = uc.uploaderService.UploadAvatarFile(ctx)
+ url, err = uc.uploaderService.UploadAvatarFile(ctx, userID)
case fileFromPost:
- url, err = uc.uploaderService.UploadPostFile(ctx)
+ url, err = uc.uploaderService.UploadPostFile(ctx, userID)
case fileFromBranding:
if !middleware.GetIsAdminFromContext(ctx) {
handler.HandleResponse(ctx, errors.Forbidden(reason.ForbiddenError), nil)
return
}
- url, err = uc.uploaderService.UploadBrandingFile(ctx)
+ url, err = uc.uploaderService.UploadBrandingFile(ctx, userID)
case fileFromPostAttachment:
- url, err = uc.uploaderService.UploadPostAttachment(ctx)
+ url, err = uc.uploaderService.UploadPostAttachment(ctx, userID)
default:
handler.HandleResponse(ctx, errors.BadRequest(reason.UploadFileSourceUnsupported), nil)
return
diff --git a/internal/entity/file_record_entity.go b/internal/entity/file_record_entity.go
new file mode 100644
index 00000000..83a917aa
--- /dev/null
+++ b/internal/entity/file_record_entity.go
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package entity
+
+import "time"
+
+const (
+ FileRecordStatusAvailable = 1
+ FileRecordStatusDeleted = 10
+)
+
+// FileRecord file record
+type FileRecord struct {
+ ID int `xorm:"not null pk autoincr INT(10) id"`
+ CreatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP created TIMESTAMP created_at"`
+ UpdatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP updated TIMESTAMP updated_at"`
+ UserID string `xorm:"not null default 0 BIGINT(20) user_id"`
+ FilePath string `xorm:"not null VARCHAR(256) file_path"`
+ FileURL string `xorm:"not null VARCHAR(1024) file_url"`
+ ObjectID string `xorm:"not null default 0 INDEX BIGINT(20) object_id"`
+ Source string `xorm:"not null VARCHAR(128) source"`
+ Status int `xorm:"not null default 0 TINYINT(4) status"`
+}
+
+// TableName file record table name
+func (FileRecord) TableName() string {
+ return "file_record"
+}
diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go
index bc2ecda7..57f4778e 100644
--- a/internal/migrations/migrations.go
+++ b/internal/migrations/migrations.go
@@ -100,6 +100,7 @@ var migrations = []Migration{
NewMigration("v1.4.0", "add badge/badge_group/badge_award table", addBadges, true),
NewMigration("v1.4.1", "add question link", addQuestionLink, true),
NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true),
+ NewMigration("v1.4.5", "add file record", addFileRecord, true),
}
func GetMigrations() []Migration {
diff --git a/internal/service/service_config/service_config.go b/internal/migrations/v25.go
similarity index 78%
copy from internal/service/service_config/service_config.go
copy to internal/migrations/v25.go
index c5f27669..a0e1f1c2 100644
--- a/internal/service/service_config/service_config.go
+++ b/internal/migrations/v25.go
@@ -17,8 +17,15 @@
* under the License.
*/
-package service_config
+package migrations
-type ServiceConfig struct {
- UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`
+import (
+ "context"
+
+ "github.com/apache/answer/internal/entity"
+ "xorm.io/xorm"
+)
+
+func addFileRecord(ctx context.Context, x *xorm.Engine) error {
+ return x.Context(ctx).Sync(new(entity.FileRecord))
}
diff --git a/internal/repo/answer/answer_repo.go b/internal/repo/answer/answer_repo.go
index 2c2787f6..cdfc5802 100644
--- a/internal/repo/answer/answer_repo.go
+++ b/internal/repo/answer/answer_repo.go
@@ -529,7 +529,24 @@ func (ar *answerRepo) updateSearch(ctx context.Context, answerID string) (err er
}
func (ar *answerRepo) DeletePermanentlyAnswers(ctx context.Context) error {
- _, err := ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})
+ // get all deleted answers ids
+ ids := make([]string, 0)
+ err := ar.data.DB.Context(ctx).Select("id").Table(new(entity.Answer).TableName()).
+ Where("status = ?", entity.AnswerStatusDeleted).Find(&ids)
+ if err != nil {
+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+
+ // delete all revisions permanently
+ _, err = ar.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})
+ if err != nil {
+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+
+ _, err = ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
diff --git a/internal/repo/file_record/file_record_repo.go b/internal/repo/file_record/file_record_repo.go
new file mode 100644
index 00000000..ed081be4
--- /dev/null
+++ b/internal/repo/file_record/file_record_repo.go
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package file_record
+
+import (
+ "context"
+
+ "github.com/apache/answer/internal/base/pager"
+ "github.com/apache/answer/internal/service/file_record"
+
+ "github.com/apache/answer/internal/base/data"
+ "github.com/apache/answer/internal/base/reason"
+ "github.com/apache/answer/internal/entity"
+ "github.com/segmentfault/pacman/errors"
+)
+
+// fileRecordRepo fileRecord repository
+type fileRecordRepo struct {
+ data *data.Data
+}
+
+// NewFileRecordRepo new repository
+func NewFileRecordRepo(data *data.Data) file_record.FileRecordRepo {
+ return &fileRecordRepo{
+ data: data,
+ }
+}
+
+// AddFileRecord add file record
+func (fr *fileRecordRepo) AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) {
+ _, err = fr.data.DB.Context(ctx).Insert(fileRecord)
+ if err != nil {
+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ return
+}
+
+// GetFileRecordPage get fileRecord page
+func (fr *fileRecordRepo) GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) (
+ fileRecordList []*entity.FileRecord, total int64, err error) {
+ fileRecordList = make([]*entity.FileRecord, 0)
+
+ session := fr.data.DB.Context(ctx)
+ total, err = pager.Help(page, pageSize, &fileRecordList, cond, session)
+ if err != nil {
+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ return
+}
+
+// DeleteFileRecord delete file record
+func (fr *fileRecordRepo) DeleteFileRecord(ctx context.Context, id int) (err error) {
+ _, err = fr.data.DB.Context(ctx).ID(id).Cols("status").Update(&entity.FileRecord{Status: entity.FileRecordStatusDeleted})
+ if err != nil {
+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ return
+}
+
+// UpdateFileRecord update file record
+func (fr *fileRecordRepo) UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) {
+ _, err = fr.data.DB.Context(ctx).ID(fileRecord.ID).Update(fileRecord)
+ if err != nil {
+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ return
+}
diff --git a/internal/repo/provider.go b/internal/repo/provider.go
index 1b29cc5e..02f27f62 100644
--- a/internal/repo/provider.go
+++ b/internal/repo/provider.go
@@ -33,6 +33,7 @@ import (
"github.com/apache/answer/internal/repo/comment"
"github.com/apache/answer/internal/repo/config"
"github.com/apache/answer/internal/repo/export"
+ "github.com/apache/answer/internal/repo/file_record"
"github.com/apache/answer/internal/repo/limit"
"github.com/apache/answer/internal/repo/meta"
"github.com/apache/answer/internal/repo/notification"
@@ -107,4 +108,5 @@ var ProviderSetRepo = wire.NewSet(
badge.NewEventRuleRepo,
badge_group.NewBadgeGroupRepo,
badge_award.NewBadgeAwardRepo,
+ file_record.NewFileRecordRepo,
)
diff --git a/internal/repo/question/question_repo.go b/internal/repo/question/question_repo.go
index 117ee38f..978c9ce3 100644
--- a/internal/repo/question/question_repo.go
+++ b/internal/repo/question/question_repo.go
@@ -168,6 +168,23 @@ func (qr *questionRepo) UpdateQuestionStatusWithOutUpdateTime(ctx context.Contex
}
func (qr *questionRepo) DeletePermanentlyQuestions(ctx context.Context) (err error) {
+ // get all deleted question ids
+ ids := make([]string, 0)
+ err = qr.data.DB.Context(ctx).Select("id").Table(new(entity.Question).TableName()).
+ Where("status = ?", entity.QuestionStatusDeleted).Find(&ids)
+ if err != nil {
+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+
+ // delete all revisions permanently
+ _, err = qr.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})
+ if err != nil {
+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+
_, err = qr.data.DB.Context(ctx).Where("status = ?", entity.QuestionStatusDeleted).Delete(&entity.Question{})
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
diff --git a/internal/repo/revision/revision_repo.go b/internal/repo/revision/revision_repo.go
index ab00adf9..09ba1aac 100644
--- a/internal/repo/revision/revision_repo.go
+++ b/internal/repo/revision/revision_repo.go
@@ -155,7 +155,17 @@ func (rr *revisionRepo) GetLastRevisionByObjectID(ctx context.Context, objectID
revision *entity.Revision, exist bool, err error,
) {
revision = &entity.Revision{}
- exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).OrderBy("created_at DESC").Get(revision)
+ exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).Desc("created_at").Get(revision)
+ if err != nil {
+ return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ }
+ return
+}
+
+// GetLastRevisionByFileURL get object's last revision by file url
+func (rr *revisionRepo) GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error) {
+ revision = &entity.Revision{}
+ exist, err = rr.data.DB.Context(ctx).Where("content LIKE ?", "%"+fileURL+"%").Desc("created_at").Get(revision)
if err != nil {
return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
diff --git a/internal/router/static_router.go b/internal/router/static_router.go
index 807f4c73..a6c80fc0 100644
--- a/internal/router/static_router.go
+++ b/internal/router/static_router.go
@@ -20,11 +20,14 @@
package router
import (
+ "net/http"
+ "path/filepath"
+ "strings"
+
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/service/service_config"
+ "github.com/apache/answer/pkg/dir"
"github.com/gin-gonic/gin"
- "path/filepath"
- "strings"
)
// StaticRouter static api router
@@ -54,6 +57,11 @@ func (a *StaticRouter) RegisterStaticRouter(r *gin.RouterGroup) {
realFilename := strings.TrimSuffix(filePath, "/"+originalFilename) + filepath.Ext(originalFilename)
// The file local path is /uploads/files/post/hash.pdf
fileLocalPath := filepath.Join(a.serviceConfig.UploadPath, constant.FilesPostSubPath, realFilename)
+ // If the file is not exist, return 404
+ if !dir.CheckFileExist(fileLocalPath) {
+ c.Redirect(http.StatusFound, "/404")
+ return
+ }
c.FileAttachment(fileLocalPath, originalFilename)
})
}
diff --git a/internal/service/content/question_service.go b/internal/service/content/question_service.go
index 109b6b10..f3641dcc 100644
--- a/internal/service/content/question_service.go
+++ b/internal/service/content/question_service.go
@@ -1398,6 +1398,8 @@ func (qs *QuestionService) GetQuestionPage(ctx context.Context, req *schema.Ques
return nil, 0, err
}
tagIDs = append(synTagIds, tagInfo.ID)
+ } else {
+ return questions, 0, nil
}
}
diff --git a/internal/service/file_record/file_record_service.go b/internal/service/file_record/file_record_service.go
new file mode 100644
index 00000000..abb98376
--- /dev/null
+++ b/internal/service/file_record/file_record_service.go
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package file_record
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/apache/answer/internal/base/constant"
+ "github.com/apache/answer/internal/entity"
+ "github.com/apache/answer/internal/service/revision"
+ "github.com/apache/answer/internal/service/service_config"
+ "github.com/apache/answer/internal/service/siteinfo_common"
+ "github.com/apache/answer/pkg/checker"
+ "github.com/apache/answer/pkg/dir"
+ "github.com/apache/answer/pkg/writer"
+ "github.com/segmentfault/pacman/log"
+)
+
+// FileRecordRepo file record repository
+type FileRecordRepo interface {
+ AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error)
+ UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error)
+ GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) (
+ fileRecordList []*entity.FileRecord, total int64, err error)
+ DeleteFileRecord(ctx context.Context, id int) (err error)
+}
+
+// FileRecordService file record service
+type FileRecordService struct {
+ fileRecordRepo FileRecordRepo
+ revisionRepo revision.RevisionRepo
+ serviceConfig *service_config.ServiceConfig
+ siteInfoService siteinfo_common.SiteInfoCommonService
+}
+
+// NewFileRecordService new file record service
+func NewFileRecordService(
+ fileRecordRepo FileRecordRepo,
+ revisionRepo revision.RevisionRepo,
+ serviceConfig *service_config.ServiceConfig,
+ siteInfoService siteinfo_common.SiteInfoCommonService,
+) *FileRecordService {
+ return &FileRecordService{
+ fileRecordRepo: fileRecordRepo,
+ revisionRepo: revisionRepo,
+ serviceConfig: serviceConfig,
+ siteInfoService: siteInfoService,
+ }
+}
+
+// AddFileRecord add file record
+func (fs *FileRecordService) AddFileRecord(ctx context.Context, userID, filePath, fileURL, source string) {
+ record := &entity.FileRecord{
+ UserID: userID,
+ FilePath: filePath,
+ FileURL: fileURL,
+ Source: source,
+ Status: entity.FileRecordStatusAvailable,
+ ObjectID: "0",
+ }
+ if err := fs.fileRecordRepo.AddFileRecord(ctx, record); err != nil {
+ log.Errorf("add file record error: %v", err)
+ }
+}
+
+// CleanOrphanUploadFiles clean orphan upload files
+func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) {
+ page, pageSize := 1, 1000
+
+ for {
+ fileRecordList, total, err := fs.fileRecordRepo.GetFileRecordPage(ctx, page, pageSize, &entity.FileRecord{
+ Status: entity.FileRecordStatusAvailable,
+ })
+ if err != nil {
+ log.Errorf("get file record page error: %v", err)
+ return
+ }
+ if len(fileRecordList) == 0 || total == 0 {
+ break
+ }
+ for _, fileRecord := range fileRecordList {
+ // If this file record created in 48 hours, no need to check
+ if fileRecord.CreatedAt.AddDate(0, 0, 2).After(time.Now()) {
+ continue
+ }
+ if checker.IsNotZeroString(fileRecord.ObjectID) {
+ _, exist, err := fs.revisionRepo.GetLastRevisionByObjectID(ctx, fileRecord.ObjectID)
+ if err != nil {
+ log.Errorf("get last revision by object id error: %v", err)
+ continue
+ }
+ if exist {
+ continue
+ }
+ } else {
+ lastRevision, exist, err := fs.revisionRepo.GetLastRevisionByFileURL(ctx, fileRecord.FileURL)
+ if err != nil {
+ log.Errorf("get last revision by file url error: %v", err)
+ continue
+ }
+ if exist {
+ // update the file record object id
+ fileRecord.ObjectID = lastRevision.ObjectID
+ if err := fs.fileRecordRepo.UpdateFileRecord(ctx, fileRecord); err != nil {
+ log.Errorf("update file record object id error: %v", err)
+ }
+ continue
+ }
+ }
+ // Delete and move the file record
+ if err := fs.deleteAndMoveFileRecord(ctx, fileRecord); err != nil {
+ log.Error(err)
+ }
+ }
+ page++
+ }
+}
+
+func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) {
+ deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath)
+ log.Infof("purge deleted files: %s", deletedPath)
+ err := os.RemoveAll(deletedPath)
+ if err != nil {
+ log.Errorf("purge deleted files error: %v", err)
+ return
+ }
+ err = dir.CreateDirIfNotExist(deletedPath)
+ if err != nil {
+ log.Errorf("create deleted directory error: %v", err)
+ }
+ return
+}
+
+func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, fileRecord *entity.FileRecord) error {
+ // Delete the file record
+ if err := fs.fileRecordRepo.DeleteFileRecord(ctx, fileRecord.ID); err != nil {
+ return fmt.Errorf("delete file record error: %v", err)
+ }
+
+ // Move the file to the deleted directory
+ oldFilename := filepath.Base(fileRecord.FilePath)
+ oldFilePath := filepath.Join(fs.serviceConfig.UploadPath, fileRecord.FilePath)
+ deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath, oldFilename)
+
+ if err := writer.MoveFile(oldFilePath, deletedPath); err != nil {
+ return fmt.Errorf("move file error: %v", err)
+ }
+
+ log.Debugf("delete and move file: %s", fileRecord.FileURL)
+ return nil
+}
diff --git a/internal/service/object_info/object_info.go b/internal/service/object_info/object_info.go
index 76b1d86d..5ef438ff 100644
--- a/internal/service/object_info/object_info.go
+++ b/internal/service/object_info/object_info.go
@@ -21,6 +21,7 @@ package object_info
import (
"context"
+
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/base/reason"
"github.com/apache/answer/internal/schema"
@@ -279,7 +280,7 @@ func (os *ObjService) GetInfo(ctx context.Context, objectID string) (objInfo *sc
ObjectID: tagInfo.ID,
TagID: tagInfo.ID,
ObjectType: objectType,
- Title: tagInfo.ParsedText,
+ Title: tagInfo.SlugName,
Content: tagInfo.ParsedText, // todo trim
}
}
diff --git a/internal/service/provider.go b/internal/service/provider.go
index 84073235..4b1b6427 100644
--- a/internal/service/provider.go
+++ b/internal/service/provider.go
@@ -36,6 +36,7 @@ import (
"github.com/apache/answer/internal/service/dashboard"
"github.com/apache/answer/internal/service/event_queue"
"github.com/apache/answer/internal/service/export"
+ "github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/service/follow"
"github.com/apache/answer/internal/service/importer"
"github.com/apache/answer/internal/service/meta"
@@ -126,4 +127,5 @@ var ProviderSetService = wire.NewSet(
badge.NewBadgeAwardService,
badge.NewBadgeGroupService,
importer.NewImporterService,
+ file_record.NewFileRecordService,
)
diff --git a/internal/service/revision/revision.go b/internal/service/revision/revision.go
index 58427220..33ce285e 100644
--- a/internal/service/revision/revision.go
+++ b/internal/service/revision/revision.go
@@ -31,6 +31,7 @@ type RevisionRepo interface {
AddRevision(ctx context.Context, revision *entity.Revision, autoUpdateRevisionID bool) (err error)
GetRevisionByID(ctx context.Context, revisionID string) (revision *entity.Revision, exist bool, err error)
GetLastRevisionByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error)
+ GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error)
GetRevisionList(ctx context.Context, revision *entity.Revision) (revisionList []entity.Revision, err error)
UpdateObjectRevisionId(ctx context.Context, revision *entity.Revision, session *xorm.Session) (err error)
ExistUnreviewedByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error)
diff --git a/internal/service/service_config/service_config.go b/internal/service/service_config/service_config.go
index c5f27669..90e399e1 100644
--- a/internal/service/service_config/service_config.go
+++ b/internal/service/service_config/service_config.go
@@ -20,5 +20,8 @@
package service_config
type ServiceConfig struct {
- UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`
+ UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`
+ CleanUpUploads bool `json:"clean_up_uploads" mapstructure:"clean_up_uploads" yaml:"clean_up_uploads"`
+ CleanOrphanUploadsPeriodHours int `json:"clean_orphan_uploads_period_hours" mapstructure:"clean_orphan_uploads_period_hours" yaml:"clean_orphan_uploads_period_hours"`
+ PurgeDeletedFilesPeriodDays int `json:"purge_deleted_files_period_days" mapstructure:"purge_deleted_files_period_days" yaml:"purge_deleted_files_period_days"`
}
diff --git a/internal/service/tag_common/tag_common.go b/internal/service/tag_common/tag_common.go
index e5c136c3..4ce89975 100644
--- a/internal/service/tag_common/tag_common.go
+++ b/internal/service/tag_common/tag_common.go
@@ -349,10 +349,17 @@ func (ts *TagCommonService) AddTag(ctx context.Context, req *schema.AddTagReq) (
}
tagInfoJson, _ := json.Marshal(tagInfo)
revisionDTO.Content = string(tagInfoJson)
- _, err = ts.revisionService.AddRevision(ctx, revisionDTO, true)
+ revisionID, err := ts.revisionService.AddRevision(ctx, revisionDTO, true)
if err != nil {
return nil, err
}
+ ts.activityQueueService.Send(ctx, &schema.ActivityMsg{
+ UserID: req.UserID,
+ ObjectID: tagInfo.ID,
+ OriginalObjectID: tagInfo.ID,
+ ActivityTypeKey: constant.ActTagCreated,
+ RevisionID: revisionID,
+ })
return &schema.AddTagResp{SlugName: tagInfo.SlugName}, nil
}
diff --git a/internal/service/uploader/upload.go b/internal/service/uploader/upload.go
index 4b018cb4..d5cc2dfb 100644
--- a/internal/service/uploader/upload.go
+++ b/internal/service/uploader/upload.go
@@ -31,6 +31,8 @@ import (
"path/filepath"
"strings"
+ "github.com/apache/answer/internal/service/file_record"
+
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/base/reason"
"github.com/apache/answer/internal/service/service_config"
@@ -53,6 +55,7 @@ var (
constant.PostSubPath,
constant.BrandingSubPath,
constant.FilesPostSubPath,
+ constant.DeletedSubPath,
}
supportedThumbFileExtMapping = map[string]imaging.Format{
".jpg": imaging.JPEG,
@@ -63,22 +66,26 @@ var (
)
type UploaderService interface {
- UploadAvatarFile(ctx *gin.Context) (url string, err error)
- UploadPostFile(ctx *gin.Context) (url string, err error)
- UploadPostAttachment(ctx *gin.Context) (url string, err error)
- UploadBrandingFile(ctx *gin.Context) (url string, err error)
+ UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error)
+ UploadPostFile(ctx *gin.Context, userID string) (url string, err error)
+ UploadPostAttachment(ctx *gin.Context, userID string) (url string, err error)
+ UploadBrandingFile(ctx *gin.Context, userID string) (url string, err error)
AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error)
}
// uploaderService uploader service
type uploaderService struct {
- serviceConfig *service_config.ServiceConfig
- siteInfoService siteinfo_common.SiteInfoCommonService
+ serviceConfig *service_config.ServiceConfig
+ siteInfoService siteinfo_common.SiteInfoCommonService
+ fileRecordService *file_record.FileRecordService
}
// NewUploaderService new upload service
-func NewUploaderService(serviceConfig *service_config.ServiceConfig,
- siteInfoService siteinfo_common.SiteInfoCommonService) UploaderService {
+func NewUploaderService(
+ serviceConfig *service_config.ServiceConfig,
+ siteInfoService siteinfo_common.SiteInfoCommonService,
+ fileRecordService *file_record.FileRecordService,
+) UploaderService {
for _, subPath := range subPathList {
err := dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath))
if err != nil {
@@ -86,13 +93,14 @@ func NewUploaderService(serviceConfig *service_config.ServiceConfig,
}
}
return &uploaderService{
- serviceConfig: serviceConfig,
- siteInfoService: siteInfoService,
+ serviceConfig: serviceConfig,
+ siteInfoService: siteInfoService,
+ fileRecordService: fileRecordService,
}
}
// UploadAvatarFile upload avatar file
-func (us *uploaderService) UploadAvatarFile(ctx *gin.Context) (url string, err error) {
+func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.UserAvatar)
if err != nil {
return "", err
@@ -174,7 +182,7 @@ func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, si
return saveFilePath, nil
}
-func (us *uploaderService) UploadPostFile(ctx *gin.Context) (
+func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) (
url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.UserPost)
if err != nil {
@@ -202,10 +210,15 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context) (
fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
avatarFilePath := path.Join(constant.PostSubPath, newFilename)
- return us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+ url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+ if err != nil {
+ return "", err
+ }
+ us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserPost))
+ return url, nil
}
-func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (
+func (us *uploaderService) UploadPostAttachment(ctx *gin.Context, userID string) (
url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.UserPostAttachment)
if err != nil {
@@ -232,11 +245,16 @@ func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (
fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
- avatarFilePath := path.Join(constant.FilesPostSubPath, newFilename)
- return us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, avatarFilePath)
+ attachmentFilePath := path.Join(constant.FilesPostSubPath, newFilename)
+ url, err = us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, attachmentFilePath)
+ if err != nil {
+ return "", err
+ }
+ us.fileRecordService.AddFileRecord(ctx, userID, attachmentFilePath, url, string(plugin.UserPostAttachment))
+ return url, nil
}
-func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) (
+func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) (
url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.AdminBranding)
if err != nil {
diff --git a/pkg/writer/writer.go b/pkg/writer/writer.go
index 49ded3df..ff08d259 100644
--- a/pkg/writer/writer.go
+++ b/pkg/writer/writer.go
@@ -48,3 +48,8 @@ func WriteFile(filePath, content string) error {
}
return nil
}
+
+// MoveFile move file to new path
+func MoveFile(oldPath, newPath string) error {
+ return os.Rename(oldPath, newPath)
+}