You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2022/08/12 20:10:35 UTC
[beam] branch master updated: [Playground] [Backend] Removing unused snippets manually and using the scheduled task (#22389)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ee59b8ff61d [Playground] [Backend] Removing unused snippets manually and using the scheduled task (#22389)
ee59b8ff61d is described below
commit ee59b8ff61d943998b6cf1fb2d9e35035c969f2e
Author: Vladislav Chunikhin <10...@users.noreply.github.com>
AuthorDate: Fri Aug 12 23:10:25 2022 +0300
[Playground] [Backend] Removing unused snippets manually and using the scheduled task (#22389)
* [Playground] [Backend] moved properties for only router
* [Playground] [Backend] minor fix for the props variable
* [Playground] [Infrastructure] added property path in Docker files
* [Playground] [Backend] added removing unused snippets
* [Playground] [Backend] edited comments
---
playground/README.md | 7 ++
playground/backend/README.md | 2 +
playground/backend/build.gradle.kts | 11 +-
playground/backend/cmd/remove_unused_snippets.go | 59 +++++++++
playground/backend/cmd/server/server.go | 17 ++-
playground/backend/go.mod | 1 +
playground/backend/go.sum | 3 +
.../backend/internal/db/datastore/datastore_db.go | 46 ++++++-
.../internal/db/datastore/datastore_db_test.go | 139 ++++++++++++++++++++-
playground/backend/internal/db/db.go | 5 +-
.../backend/{go.mod => internal/db/dto/snippet.go} | 23 +---
.../backend/internal/environment/property.go | 6 +-
.../backend/internal/environment/property_test.go | 4 +-
playground/backend/internal/tasks/task.go | 56 +++++++++
playground/backend/properties.yaml | 6 +
15 files changed, 350 insertions(+), 35 deletions(-)
diff --git a/playground/README.md b/playground/README.md
index 652244a86de..11e17ff0322 100644
--- a/playground/README.md
+++ b/playground/README.md
@@ -74,6 +74,13 @@ cd beam
./gradlew playground:backend:containers:router:dockerComposeLocalDown
```
+## Run the method to remove unused code snippets from the Cloud Datastore. Unused snippets are snippets that are out of date. If the last visited date property less or equals than the current date minus dayDiff parameter then a snippet is out of date
+
+```
+cd beam
+./gradlew playground:backend:removeUnusedSnippet -DdayDiff={int} -DprojectId={string}
+```
+
# Deployment
See [terraform](./terraform/README.md) for details on how to build and deploy
diff --git a/playground/backend/README.md b/playground/backend/README.md
index 6ff3506495a..8d847d2569e 100644
--- a/playground/backend/README.md
+++ b/playground/backend/README.md
@@ -102,6 +102,8 @@ These properties are stored in `backend/properties.yaml` file:
- `playground_salt` - is the salt to generate the hash to avoid whatever problems a collision may cause.
- `max_snippet_size` - is the file content size limit. Since 1 character occupies 1 byte of memory, and 1 MB is approximately equal to 1000000 bytes, then maximum size of the snippet is 1000000.
- `id_length` - is the length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity.
+- `removing_unused_snippets_cron` - is the cron expression for the scheduled task to remove unused snippets.
+- `removing_unused_snippets_days` - is the number of days after which a snippet becomes unused.
### Running the server app via Docker
diff --git a/playground/backend/build.gradle.kts b/playground/backend/build.gradle.kts
index ad279993051..04d82a1d526 100644
--- a/playground/backend/build.gradle.kts
+++ b/playground/backend/build.gradle.kts
@@ -79,6 +79,15 @@ val test by tasks.registering {
test { dependsOn(startDatastoreEmulator) }
test { finalizedBy(stopDatastoreEmulator) }
+task("removeUnusedSnippet") {
+ doLast {
+ exec {
+ executable("go")
+ args("run", "cmd/remove_unused_snippets.go", System.getProperty("dayDiff"), System.getProperty("projectId"))
+ }
+ }
+}
+
task("benchmarkPrecompiledObjects") {
group = "verification"
description = "Run benchmarks for precompiled objects"
@@ -121,7 +130,7 @@ task("runLint") {
doLast {
exec {
executable("golangci-lint")
- args("run", "cmd/server/...")
+ args("run", "cmd/server/...")
}
}
}
diff --git a/playground/backend/cmd/remove_unused_snippets.go b/playground/backend/cmd/remove_unused_snippets.go
new file mode 100644
index 00000000000..ab875f3f400
--- /dev/null
+++ b/playground/backend/cmd/remove_unused_snippets.go
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strconv"
+ "time"
+
+ "beam.apache.org/playground/backend/internal/db/datastore"
+)
+
+func main() {
+ if len(os.Args) < 3 {
+ fmt.Println("Go must have at least three arguments")
+ return
+ }
+ dayDiff := os.Args[1]
+ projectId := os.Args[2]
+ fmt.Println("Removing unused snippets is running...")
+ startDate := time.Now()
+
+ diff, err := strconv.Atoi(dayDiff)
+ if err != nil {
+ fmt.Printf("Couldn't convert days to integer from the input parameter, err: %s \n", err.Error())
+ return
+ }
+
+ ctx := context.Background()
+ db, err := datastore.New(ctx, projectId)
+ if err != nil {
+ fmt.Printf("Couldn't create the database client, err: %s \n", err.Error())
+ return
+ }
+
+ err = db.DeleteUnusedSnippets(ctx, int32(diff))
+ if err != nil {
+ fmt.Printf("Couldn't delete unused code snippets, err: %s \n", err.Error())
+ return
+ }
+
+ diffTime := time.Now().Sub(startDate).Milliseconds()
+ fmt.Printf("Removing unused snippets finished, work time: %d ms\n", diffTime)
+}
diff --git a/playground/backend/cmd/server/server.go b/playground/backend/cmd/server/server.go
index faa7783ac83..fe4787ffb11 100644
--- a/playground/backend/cmd/server/server.go
+++ b/playground/backend/cmd/server/server.go
@@ -16,6 +16,13 @@
package main
import (
+ "context"
+ "fmt"
+
+ "beam.apache.org/playground/backend/internal/tasks"
+ "github.com/improbable-eng/grpc-web/go/grpcweb"
+ "google.golang.org/grpc"
+
pb "beam.apache.org/playground/backend/internal/api/v1"
"beam.apache.org/playground/backend/internal/cache"
"beam.apache.org/playground/backend/internal/cache/local"
@@ -29,10 +36,6 @@ import (
"beam.apache.org/playground/backend/internal/environment"
"beam.apache.org/playground/backend/internal/logger"
"beam.apache.org/playground/backend/internal/utils"
- "context"
- "fmt"
- "github.com/improbable-eng/grpc-web/go/grpcweb"
- "google.golang.org/grpc"
)
// runServer is starting http server wrapped on grpc
@@ -81,6 +84,12 @@ func runServer() error {
}
entityMapper = mapper.New(&envService.ApplicationEnvs, props)
+
+ // Since only router server has the scheduled task, the task creation is here
+ scheduledTasks := tasks.New(ctx)
+ if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, props.RemovingUnusedSnptsDays, dbClient); err != nil {
+ return err
+ }
}
pb.RegisterPlaygroundServiceServer(grpcServer, &playgroundController{
diff --git a/playground/backend/go.mod b/playground/backend/go.mod
index 8db8e8fc3c7..e723ea4ab94 100644
--- a/playground/backend/go.mod
+++ b/playground/backend/go.mod
@@ -25,6 +25,7 @@ require (
github.com/go-redis/redismock/v8 v8.0.6
github.com/google/uuid v1.3.0
github.com/improbable-eng/grpc-web v0.14.1
+ github.com/procyon-projects/chrono v1.1.0
github.com/rs/cors v1.8.0
github.com/spf13/viper v1.12.0
go.uber.org/goleak v1.1.12
diff --git a/playground/backend/go.sum b/playground/backend/go.sum
index 524c2590997..d93529bab9c 100644
--- a/playground/backend/go.sum
+++ b/playground/backend/go.sum
@@ -502,6 +502,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
+github.com/procyon-projects/chrono v1.1.0 h1:ZE2gnfHuhq/3Sepappk7bvObXpjoEu6uzeX6HMFOHMc=
+github.com/procyon-projects/chrono v1.1.0/go.mod h1:RwQ27W7hRaq+QUWN2yXU3BDG2FUyEQiKds8/M1FI5C8=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -569,6 +571,7 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
diff --git a/playground/backend/internal/db/datastore/datastore_db.go b/playground/backend/internal/db/datastore/datastore_db.go
index dbdd18a280e..b31882d8553 100644
--- a/playground/backend/internal/db/datastore/datastore_db.go
+++ b/playground/backend/internal/db/datastore/datastore_db.go
@@ -16,13 +16,16 @@
package datastore
import (
- "beam.apache.org/playground/backend/internal/db/entity"
- "beam.apache.org/playground/backend/internal/logger"
- "beam.apache.org/playground/backend/internal/utils"
- "cloud.google.com/go/datastore"
"context"
"fmt"
"time"
+
+ "beam.apache.org/playground/backend/internal/db/dto"
+ "cloud.google.com/go/datastore"
+
+ "beam.apache.org/playground/backend/internal/db/entity"
+ "beam.apache.org/playground/backend/internal/logger"
+ "beam.apache.org/playground/backend/internal/utils"
)
const (
@@ -194,3 +197,38 @@ func (d *Datastore) GetSDK(ctx context.Context, id string) (*entity.SDKEntity, e
}
return sdk, nil
}
+
+//DeleteUnusedSnippets deletes all unused snippets
+func (d *Datastore) DeleteUnusedSnippets(ctx context.Context, dayDiff int32) error {
+ var hoursDiff = dayDiff * 24
+ boundaryDate := time.Now().Add(-time.Hour * time.Duration(hoursDiff))
+ snippetQuery := datastore.NewQuery(SnippetKind).
+ Namespace(Namespace).
+ Filter("lVisited <= ", boundaryDate).
+ Filter("origin =", "PG_USER").
+ Project("numberOfFiles")
+ var snpDtos []*dto.SnippetDeleteDTO
+ snpKeys, err := d.Client.GetAll(ctx, snippetQuery, &snpDtos)
+ if err != nil {
+ logger.Errorf("Datastore: DeleteUnusedSnippets(): error during deleting unused snippets, err: %s\n", err.Error())
+ return err
+ }
+ var fileKeys []*datastore.Key
+ for snpIndex, snpKey := range snpKeys {
+ for fileIndex := 0; fileIndex < snpDtos[snpIndex].NumberOfFiles; fileIndex++ {
+ fileId := fmt.Sprintf("%s_%d", snpKey.Name, fileIndex)
+ fileKey := utils.GetNameKey(FileKind, fileId, Namespace, nil)
+ fileKeys = append(fileKeys, fileKey)
+ }
+ }
+ _, err = d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
+ err = tx.DeleteMulti(fileKeys)
+ err = tx.DeleteMulti(snpKeys)
+ return err
+ })
+ if err != nil {
+ logger.Errorf("Datastore: DeleteUnusedSnippets(): error during deleting unused snippets, err: %s\n", err.Error())
+ return err
+ }
+ return nil
+}
diff --git a/playground/backend/internal/db/datastore/datastore_db_test.go b/playground/backend/internal/db/datastore/datastore_db_test.go
index 5148980def2..799c6b1fa32 100644
--- a/playground/backend/internal/db/datastore/datastore_db_test.go
+++ b/playground/backend/internal/db/datastore/datastore_db_test.go
@@ -16,14 +16,17 @@
package datastore
import (
- pb "beam.apache.org/playground/backend/internal/api/v1"
- "beam.apache.org/playground/backend/internal/db/entity"
- "beam.apache.org/playground/backend/internal/utils"
- "cloud.google.com/go/datastore"
"context"
+ "fmt"
"os"
"testing"
"time"
+
+ "cloud.google.com/go/datastore"
+
+ pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/db/entity"
+ "beam.apache.org/playground/backend/internal/utils"
)
const (
@@ -378,6 +381,112 @@ func TestDatastore_GetSDK(t *testing.T) {
}
}
+func TestDatastore_DeleteUnusedSnippets(t *testing.T) {
+ type args struct {
+ ctx context.Context
+ dayDiff int32
+ }
+ now := time.Now()
+ tests := []struct {
+ name string
+ args args
+ prepare func()
+ wantErr bool
+ }{
+ {
+ name: "DeleteUnusedSnippets() with different cases",
+ args: args{ctx: ctx, dayDiff: 10},
+ prepare: func() {
+ //last visit date is now - 7 days
+ putSnippet("MOCK_ID0", "PG_USER", now.Add(-time.Hour*24*7), 2)
+ //last visit date is now - 10 days
+ putSnippet("MOCK_ID1", "PG_USER", now.Add(-time.Hour*24*10), 4)
+ //last visit date is now - 15 days
+ putSnippet("MOCK_ID2", "PG_USER", now.Add(-time.Hour*24*15), 8)
+ //last visit date is now
+ putSnippet("MOCK_ID3", "PG_USER", now, 1)
+ //last visit date is now + 2 days
+ putSnippet("MOCK_ID4", "PG_USER", now.Add(time.Hour*24*2), 2)
+ //last visit date is now + 10 days
+ putSnippet("MOCK_ID5", "PG_USER", now.Add(time.Hour*24*10), 2)
+ //last visit date is now - 18 days
+ putSnippet("MOCK_ID6", "PG_USER", now.Add(-time.Hour*24*18), 3)
+ //last visit date is now - 18 days and origin != PG_USER
+ putSnippet("MOCK_ID7", "PG_EXAMPLES", now.Add(-time.Hour*24*18), 2)
+ //last visit date is now - 9 days
+ putSnippet("MOCK_ID8", "PG_USER", now.Add(-time.Hour*24*9), 2)
+ //last visit date is now - 11 days
+ putSnippet("MOCK_ID9", "PG_USER", now.Add(-time.Hour*24*11), 2)
+ },
+ wantErr: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.prepare()
+ err := datastoreDb.DeleteUnusedSnippets(tt.args.ctx, tt.args.dayDiff)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("DeleteUnusedSnippets() error = %v, wantErr %v", err, tt.wantErr)
+ }
+
+ if err == nil {
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID0")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID0", 2)
+ if err != nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet shouldn't be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID1")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID1", 4)
+ if err == nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet should be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID2")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID2", 8)
+ if err == nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet should be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID3")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID3", 1)
+ if err != nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet shouldn't be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID4")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID4", 2)
+ if err != nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet shouldn't be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID5")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID5", 2)
+ if err != nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet shouldn't be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID6")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID6", 3)
+ if err == nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet should be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID7")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID7", 2)
+ if err != nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet shouldn't be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID8")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID8", 2)
+ if err != nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet shouldn't be deleted, err: %s", err)
+ }
+ _, err = datastoreDb.GetSnippet(tt.args.ctx, "MOCK_ID9")
+ _, err = datastoreDb.GetFiles(tt.args.ctx, "MOCK_ID9", 2)
+ if err == nil {
+ t.Errorf("DeleteUnusedSnippets() this snippet should be deleted, err: %s", err)
+ }
+ }
+
+ })
+ }
+}
+
func TestNew(t *testing.T) {
type args struct {
ctx context.Context
@@ -426,3 +535,25 @@ func getSDKs() []*entity.SDKEntity {
}
return sdkEntities
}
+
+func putSnippet(id, origin string, lVisited time.Time, numberOfFiles int) {
+ var files []*entity.FileEntity
+ for i := 0; i < numberOfFiles; i++ {
+ file := &entity.FileEntity{
+ Name: fmt.Sprintf("%s_%d", "MOCK_NAME", i),
+ Content: fmt.Sprintf("%s_%d", "MOCK_CONTENT", i),
+ }
+ files = append(files, file)
+ }
+ _ = datastoreDb.PutSnippet(ctx, id, &entity.Snippet{
+ IDMeta: &entity.IDMeta{Salt: "MOCK_SALT", IdLength: 11},
+ Snippet: &entity.SnippetEntity{
+ Sdk: utils.GetNameKey(SdkKind, pb.Sdk_SDK_GO.String(), Namespace, nil),
+ PipeOpts: "MOCK_OPTIONS",
+ LVisited: lVisited,
+ Origin: origin,
+ NumberOfFiles: numberOfFiles,
+ },
+ Files: files,
+ })
+}
diff --git a/playground/backend/internal/db/db.go b/playground/backend/internal/db/db.go
index a5d7dae7800..5af30f1b264 100644
--- a/playground/backend/internal/db/db.go
+++ b/playground/backend/internal/db/db.go
@@ -16,8 +16,9 @@
package db
import (
- "beam.apache.org/playground/backend/internal/db/entity"
"context"
+
+ "beam.apache.org/playground/backend/internal/db/entity"
)
type Database interface {
@@ -31,6 +32,8 @@ type SnippetDatabase interface {
GetSnippet(ctx context.Context, id string) (*entity.SnippetEntity, error)
GetFiles(ctx context.Context, snipId string, numberOfFiles int) ([]*entity.FileEntity, error)
+
+ DeleteUnusedSnippets(ctx context.Context, dayDiff int32) error
}
type CatalogDatabase interface {
diff --git a/playground/backend/go.mod b/playground/backend/internal/db/dto/snippet.go
similarity index 59%
copy from playground/backend/go.mod
copy to playground/backend/internal/db/dto/snippet.go
index 8db8e8fc3c7..8007a137161 100644
--- a/playground/backend/go.mod
+++ b/playground/backend/internal/db/dto/snippet.go
@@ -13,23 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module beam.apache.org/playground/backend
+package dto
-go 1.16
-
-require (
- cloud.google.com/go/datastore v1.6.0
- cloud.google.com/go/logging v1.4.2
- cloud.google.com/go/storage v1.23.0
- github.com/go-redis/redis/v8 v8.11.4
- github.com/go-redis/redismock/v8 v8.0.6
- github.com/google/uuid v1.3.0
- github.com/improbable-eng/grpc-web v0.14.1
- github.com/rs/cors v1.8.0
- github.com/spf13/viper v1.12.0
- go.uber.org/goleak v1.1.12
- google.golang.org/api v0.85.0
- google.golang.org/grpc v1.47.0
- google.golang.org/protobuf v1.28.0
- gopkg.in/yaml.v3 v3.0.1
-)
+type SnippetDeleteDTO struct {
+ NumberOfFiles int `datastore:"numberOfFiles"`
+}
diff --git a/playground/backend/internal/environment/property.go b/playground/backend/internal/environment/property.go
index d5cac4276d0..b4a5407e199 100644
--- a/playground/backend/internal/environment/property.go
+++ b/playground/backend/internal/environment/property.go
@@ -30,8 +30,12 @@ type Properties struct {
Salt string `mapstructure:"playground_salt"`
// MaxSnippetSize is the file content size limit. Since 1 character occupies 1 byte of memory, and 1 MB is approximately equal to 1000000 bytes, then maximum size of the snippet is 1000000.
MaxSnippetSize int32 `mapstructure:"max_snippet_size"`
- // ID_LENGTH is the length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity.
+ // IdLength is the length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity.
IdLength int8 `mapstructure:"id_length"`
+ // RemovingUnusedSnptsCron is the cron expression for the scheduled task to remove unused snippets
+ RemovingUnusedSnptsCron string `mapstructure:"removing_unused_snippets_cron"`
+ // RemovingUnusedSnptsDays is the number of days after which a snippet becomes unused
+ RemovingUnusedSnptsDays int32 `mapstructure:"removing_unused_snippets_days"`
}
func NewProperties(configPath string) (*Properties, error) {
diff --git a/playground/backend/internal/environment/property_test.go b/playground/backend/internal/environment/property_test.go
index 206d03f0ba6..26d6d7c26e2 100644
--- a/playground/backend/internal/environment/property_test.go
+++ b/playground/backend/internal/environment/property_test.go
@@ -46,7 +46,9 @@ func TestNew(t *testing.T) {
}
if props.Salt != "Beam playground salt" ||
props.MaxSnippetSize != 1000000 ||
- props.IdLength != 11 {
+ props.IdLength != 11 ||
+ props.RemovingUnusedSnptsCron != "0 0 0 1 */1 *" ||
+ props.RemovingUnusedSnptsDays != 180 {
t.Errorf("NewProperties(): unexpected result")
}
})
diff --git a/playground/backend/internal/tasks/task.go b/playground/backend/internal/tasks/task.go
new file mode 100644
index 00000000000..c77b4a06133
--- /dev/null
+++ b/playground/backend/internal/tasks/task.go
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+ "context"
+ "time"
+
+ "github.com/procyon-projects/chrono"
+
+ "beam.apache.org/playground/backend/internal/db"
+ "beam.apache.org/playground/backend/internal/logger"
+)
+
+type ScheduledTask struct {
+ ctx context.Context
+ taskScheduler chrono.TaskScheduler
+}
+
+func New(ctx context.Context) *ScheduledTask {
+ return &ScheduledTask{ctx: ctx, taskScheduler: chrono.NewDefaultTaskScheduler()}
+}
+
+func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, dayDiff int32, db db.Database) error {
+ task, err := st.taskScheduler.ScheduleWithCron(func(ctx context.Context) {
+ logger.Info("ScheduledTask: StartRemovingExtraSnippets() is running...\n")
+ startDate := time.Now()
+ if err := db.DeleteUnusedSnippets(ctx, dayDiff); err != nil {
+ logger.Errorf("ScheduledTask: StartRemovingExtraSnippets() error during deleting unused snippets, err: %s\n", err.Error())
+ }
+ diffTime := time.Now().Sub(startDate).Milliseconds()
+ logger.Infof("ScheduledTask: StartRemovingExtraSnippets() finished, work time: %d ms\n", diffTime)
+ }, cron, chrono.WithLocation("UTC"))
+
+ if err != nil {
+ logger.Errorf("ScheduledTask: StartRemovingExtraSnippets() error during task running. Task will be cancelled, err: %s\n", err.Error())
+ if !task.IsCancelled() {
+ task.Cancel()
+ }
+ return err
+ }
+ return nil
+}
diff --git a/playground/backend/properties.yaml b/playground/backend/properties.yaml
index 9c81473cb16..12f5adb2218 100644
--- a/playground/backend/properties.yaml
+++ b/playground/backend/properties.yaml
@@ -19,3 +19,9 @@ playground_salt: Beam playground salt
max_snippet_size: 1000000
# Length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity
id_length: 11
+# Cron expression for the scheduled task to remove unused snippets. Cron time: At 12:00 AM, on day 1 of the month, every month
+# Check your cron expression here: https://crontab.cronhub.io/
+# Time trigger for the scheduled task will be updated every time the application is restarted
+removing_unused_snippets_cron: 0 0 0 1 */1 *
+# Number of days after which a snippet becomes unused
+removing_unused_snippets_days: 180