You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/05/03 18:51:31 UTC
[beam] branch master updated: [Playground] Move modify saved snippets functionality to to cloudfunctions (#26026)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 68ed7ce48b2 [Playground] Move modify saved snippets functionality to to cloudfunctions (#26026)
68ed7ce48b2 is described below
commit 68ed7ce48b2dbe779c2446391aaafd48b8de9b22
Author: Timur Sultanov <ti...@akvelon.com>
AuthorDate: Wed May 3 22:51:25 2023 +0400
[Playground] Move modify saved snippets functionality to to cloudfunctions (#26026)
* Move priviledged Datastore operations to Cloudfunctions
* Terraform changes to deploy cloud functions
* Remove setting for snippets retention period as it's now hardcoded in a cloud function
* service account access review
* Datastore namespace added
* Update .gitignore
* Terraform fixes to permissions and namespaces
* Move PutSnippet() into a separate CloudFunction
* Remove unused role
* Do not apply data migrations which are already applied
* Move DB migration to external command
* Hook up DB migration script to GKE deployment task
* Remove trailing whitespace
* Apply migrations in ToB integration tests
* CloudFunctions review
* remove unused code
* Fix EOL on EOF
* Update Readme
* 'Cloud Functions Developer' role added to requirements
* Fix issue with namespace value being ignored
* Paths for archive_file set relative to path.root
* Pass namespace into cloudfunctions
* change arcive
* fix name
* fix module
* Cloud Datastore User role added
Required to run DB migration tool
* Variable renamed
---------
Co-authored-by: Sergey Makarkin <ya...@mail.ru>
Co-authored-by: Sergey Makarkin <se...@akvelon.com>
Co-authored-by: Rouslan <11...@users.noreply.github.com>
Co-authored-by: rshamunov <ru...@akvelon.com>
---
.gitignore | 1 +
learning/tour-of-beam/backend/docker-compose.yml | 1 +
playground/backend/README.md | 1 -
playground/backend/build.gradle.kts | 4 +-
.../backend/cmd/migration_tool/migration_tool.go | 56 ++++++++
.../remove_unused_snippets.go | 5 +-
playground/backend/cmd/server/controller_test.go | 21 +--
playground/backend/cmd/server/server.go | 37 ++---
playground/backend/containers/router/Dockerfile | 5 +
.../containers/router/docker-compose.local.yml | 1 +
playground/backend/containers/router/entrypoint.sh | 15 ++
playground/backend/functions.go | 118 +++++++++++++++
playground/backend/go.mod | 11 +-
playground/backend/go.sum | 30 +++-
.../backend/internal/db/datastore/datastore_db.go | 136 +++++++++++-------
.../internal/db/datastore/datastore_db_test.go | 51 +------
.../internal/db/datastore/emulator_wrapper.go | 2 +-
.../migration_base.go} | 28 ++--
.../backend/internal/db/datastore/migration_db.go | 157 ++++++++++++++++++++
playground/backend/internal/db/db.go | 15 +-
playground/backend/internal/db/entity/schema.go | 3 +-
.../internal/db/mapper/datastore_mapper_test.go | 6 +-
.../db/schema/migration/migrations_test.go | 121 ----------------
.../db/schema/{migration => }/migration_v001.go | 36 +++--
.../db/{entity/schema.go => schema/migrations.go} | 9 +-
playground/backend/internal/db/schema/version.go | 62 --------
.../backend/internal/environment/application.go | 67 +++++++--
.../internal/environment/environment_service.go | 112 +++++++++------
.../environment/environment_service_test.go | 160 +++++++++++++++++++--
.../backend/internal/environment/property.go | 2 -
.../backend/internal/environment/property_test.go | 3 +-
.../external_functions_component.go | 127 ++++++++++++++++
playground/backend/internal/tasks/task.go | 6 +-
.../backend/internal/utils/datastore_utils.go | 7 +-
playground/backend/playground_functions/Dockerfile | 38 +++++
.../schema.go => playground_functions/cmd/main.go} | 21 ++-
.../func_enviornment.go} | 23 ++-
.../middleware.go} | 17 ++-
playground/backend/properties.yaml | 2 -
playground/docker-compose.local.yaml | 43 ++++++
playground/index.yaml | 1 -
.../templates/deployment-router.yml | 6 +
playground/terraform/README.md | 8 +-
playground/terraform/build.gradle.kts | 79 +++++++++-
.../infrastructure/api_enable/variables.tf | 2 +-
.../{ip_address => archive_file}/main.tf | 10 +-
.../infrastructure/cloudfunctions/main.tf | 65 +++++++++
.../variables.tf => cloudfunctions/output.tf} | 13 +-
.../{api_enable => cloudfunctions}/variables.tf | 24 +++-
.../variables.tf => gke_bucket/main.tf} | 18 ++-
.../{ip_address/main.tf => gke_bucket/output.tf} | 4 +-
.../{ip_address => gke_bucket}/variables.tf | 9 +-
.../terraform/infrastructure/ip_address/main.tf | 2 +-
.../infrastructure/ip_address/variables.tf | 2 +-
playground/terraform/infrastructure/main.tf | 30 +++-
playground/terraform/infrastructure/output.tf | 14 +-
playground/terraform/infrastructure/setup/iam.tf | 18 ++-
.../terraform/infrastructure/setup/output.tf | 6 +-
playground/terraform/infrastructure/variables.tf | 6 +-
playground/terraform/main.tf | 6 +-
playground/terraform/output.tf | 14 +-
playground/terraform/variables.tf | 7 +-
62 files changed, 1399 insertions(+), 505 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5c106839945..5046b64fb4b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -136,6 +136,7 @@ playground/frontend/playground_components_dev/pubspec.lock
**/*.tfstate.*
**/*.hcl
**/*.tfvars
+playground/cloudfunction.zip
# Ignore Katas auto-generated files
**/*-remote-info.yaml
diff --git a/learning/tour-of-beam/backend/docker-compose.yml b/learning/tour-of-beam/backend/docker-compose.yml
index 50823ab1bfe..d47db7e020b 100644
--- a/learning/tour-of-beam/backend/docker-compose.yml
+++ b/learning/tour-of-beam/backend/docker-compose.yml
@@ -42,6 +42,7 @@ services:
- CACHE_TYPE=local
- SDK_CONFIG=/opt/playground/backend/sdks-emulator.yaml
- PROTOCOL_TYPE=TCP
+ - APPLY_MIGRATIONS=True
ports:
- "8000:8080"
depends_on:
diff --git a/playground/backend/README.md b/playground/backend/README.md
index 3a3759de538..8afb891b548 100644
--- a/playground/backend/README.md
+++ b/playground/backend/README.md
@@ -129,7 +129,6 @@ These properties are stored in `backend/properties.yaml` file:
- `max_snippet_size` - is the file content size limit. Since 1 character occupies 1 byte of memory, and 1 MB is approximately equal to 1000000 bytes, then maximum size of the snippet is 1000000.
- `id_length` - is the length of the identifier that is used to store data in the cloud datastore. It's appropriate length to save storage size in the cloud datastore and provide good randomnicity.
- `removing_unused_snippets_cron` - is the cron expression for the scheduled task to remove unused snippets.
-- `removing_unused_snippets_days` - is the number of days after which a snippet becomes unused.
## Running the server app via Docker
diff --git a/playground/backend/build.gradle.kts b/playground/backend/build.gradle.kts
index ba3c4c5feb4..03060bd61dd 100644
--- a/playground/backend/build.gradle.kts
+++ b/playground/backend/build.gradle.kts
@@ -72,7 +72,7 @@ task("removeUnusedSnippet") {
doLast {
exec {
executable("go")
- args("run", "cmd/remove_unused_snippets.go", "cleanup",
+ args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "cleanup",
"-day_diff", System.getProperty("dayDiff"), "-project_id", System.getProperty("projectId"),
"-namespace", System.getProperty("namespace"))
}
@@ -83,7 +83,7 @@ task("removeSnippet") {
doLast {
exec {
executable("go")
- args("run", "cmd/remove_unused_snippets.go", "remove",
+ args("run", "cmd/remove_unused_snippets/remove_unused_snippets.go", "remove",
"-snippet_id", System.getProperty("snippetId"), "-project_id", System.getProperty("projectId"),
"-namespace", System.getProperty("namespace"))
}
diff --git a/playground/backend/cmd/migration_tool/migration_tool.go b/playground/backend/cmd/migration_tool/migration_tool.go
new file mode 100644
index 00000000000..5553f139c78
--- /dev/null
+++ b/playground/backend/cmd/migration_tool/migration_tool.go
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "beam.apache.org/playground/backend/internal/constants"
+ "beam.apache.org/playground/backend/internal/db/datastore"
+ "beam.apache.org/playground/backend/internal/db/mapper"
+ "beam.apache.org/playground/backend/internal/db/schema"
+ "beam.apache.org/playground/backend/internal/logger"
+ "context"
+ "flag"
+ "fmt"
+ "os"
+)
+
+func main() {
+ projectId := flag.String("project-id", "", "GCP project id")
+ sdkConfigPath := flag.String("sdk-config", "", "Path to the sdk config file")
+ namespace := flag.String("namespace", constants.Namespace, "Datastore namespace")
+
+ flag.Parse()
+
+ ctx := context.WithValue(context.Background(), constants.DatastoreNamespaceKey, *namespace)
+
+ cwd, err := os.Getwd()
+ if err != nil {
+ fmt.Printf("Couldn't get the current working directory, err: %s \n", err.Error())
+ os.Exit(1)
+ }
+ logger.SetupLogger(context.Background(), cwd, *projectId)
+
+ migratedDb, err := datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), nil, *projectId)
+ if err != nil {
+ logger.Fatalf("Couldn't create DB client instance, err: %s \n", err.Error())
+ os.Exit(1)
+ }
+
+ if err := migratedDb.ApplyMigrations(ctx, schema.Migrations, *sdkConfigPath); err != nil {
+ logger.Fatalf("Couldn't apply migrations, err: %s \n", err.Error())
+ os.Exit(1)
+ }
+}
diff --git a/playground/backend/cmd/remove_unused_snippets.go b/playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go
similarity index 96%
rename from playground/backend/cmd/remove_unused_snippets.go
rename to playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go
index 616150feb4a..d5df9fcbfcd 100644
--- a/playground/backend/cmd/remove_unused_snippets.go
+++ b/playground/backend/cmd/remove_unused_snippets/remove_unused_snippets.go
@@ -30,7 +30,7 @@ import (
func createDatastoreClient(ctx context.Context, projectId string) (*datastore.Datastore, error) {
pcMapper := mapper.NewPrecompiledObjectMapper()
- db, err := datastore.New(ctx, pcMapper, projectId)
+ db, err := datastore.New(ctx, pcMapper, nil, projectId)
if err != nil {
logger.Errorf("Couldn't create the database client, err: %s \n", err.Error())
return nil, err
@@ -49,7 +49,8 @@ func cleanup(dayDiff int, projectId, namespace string) error {
return err
}
- err = db.DeleteUnusedSnippets(ctx, int32(dayDiff))
+ retentionPeriod := time.Duration(dayDiff) * 24 * time.Hour
+ err = db.DeleteUnusedSnippets(ctx, retentionPeriod)
if err != nil {
logger.Errorf("Couldn't delete unused code snippets, err: %s \n", err.Error())
return err
diff --git a/playground/backend/cmd/server/controller_test.go b/playground/backend/cmd/server/controller_test.go
index b4812adba1b..0050fa662e4 100644
--- a/playground/backend/cmd/server/controller_test.go
+++ b/playground/backend/cmd/server/controller_test.go
@@ -15,6 +15,7 @@
package main
import (
+ "beam.apache.org/playground/backend/internal/db/schema"
"context"
"fmt"
"io/fs"
@@ -42,8 +43,6 @@ import (
datastoreDb "beam.apache.org/playground/backend/internal/db/datastore"
"beam.apache.org/playground/backend/internal/db/entity"
"beam.apache.org/playground/backend/internal/db/mapper"
- "beam.apache.org/playground/backend/internal/db/schema"
- "beam.apache.org/playground/backend/internal/db/schema/migration"
"beam.apache.org/playground/backend/internal/environment"
"beam.apache.org/playground/backend/internal/logger"
"beam.apache.org/playground/backend/internal/tests/test_cleaner"
@@ -116,7 +115,9 @@ func setupServer(sdk pb.Sdk) *grpc.Server {
if err = os.Setenv("APP_WORK_DIR", path); err != nil {
panic(err)
}
- if err = os.Setenv("SDK_CONFIG", "../../../sdks-emulator.yaml"); err != nil {
+
+ sdkConfigPath := "../../../sdks-emulator.yaml"
+ if err = os.Setenv("SDK_CONFIG", sdkConfigPath); err != nil {
panic(err)
}
if err = os.Setenv("PROPERTY_PATH", "../../."); err != nil {
@@ -145,17 +146,17 @@ func setupServer(sdk pb.Sdk) *grpc.Server {
panic(err)
}
- // setup initial data
- versions := []schema.Version{
- new(migration.InitialStructure),
- new(migration.AddingComplexityProperty),
+ err = dbEmulator.ApplyMigrations(ctx, schema.Migrations, sdkConfigPath)
+ if err != nil {
+ panic(err)
}
- dbSchema := schema.New(ctx, dbEmulator, appEnv, props, versions)
- actualSchemaVersion, err := dbSchema.InitiateData()
+
+ migrationVersion, err := dbEmulator.GetCurrentDbMigrationVersion(ctx)
if err != nil {
panic(err)
}
- appEnv.SetSchemaVersion(actualSchemaVersion)
+
+ appEnv.SetSchemaVersion(migrationVersion)
// download test data to the Datastore Emulator
test_data.DownloadCatalogsWithMockData(ctx)
diff --git a/playground/backend/cmd/server/server.go b/playground/backend/cmd/server/server.go
index 9f45a177186..cdac6a440f7 100644
--- a/playground/backend/cmd/server/server.go
+++ b/playground/backend/cmd/server/server.go
@@ -16,12 +16,12 @@
package main
import (
+ "beam.apache.org/playground/backend/internal/external_functions"
"context"
"fmt"
- "os"
-
"github.com/improbable-eng/grpc-web/go/grpcweb"
"google.golang.org/grpc"
+ "os"
pb "beam.apache.org/playground/backend/internal/api/v1"
"beam.apache.org/playground/backend/internal/cache"
@@ -32,8 +32,6 @@ import (
"beam.apache.org/playground/backend/internal/db/datastore"
"beam.apache.org/playground/backend/internal/db/entity"
"beam.apache.org/playground/backend/internal/db/mapper"
- "beam.apache.org/playground/backend/internal/db/schema"
- "beam.apache.org/playground/backend/internal/db/schema/migration"
"beam.apache.org/playground/backend/internal/environment"
"beam.apache.org/playground/backend/internal/logger"
"beam.apache.org/playground/backend/internal/tasks"
@@ -67,22 +65,27 @@ func runServer() error {
// Examples catalog should be retrieved and saved to cache only if the server doesn't suppose to run code, i.e. SDK is unspecified
// Database setup only if the server doesn't suppose to run code, i.e. SDK is unspecified
if envService.BeamSdkEnvs.ApacheBeamSdk == pb.Sdk_SDK_UNSPECIFIED {
+ externalFunctions := external_functions.NewExternalFunctionsComponent(envService.ApplicationEnvs)
+
props, err = environment.NewProperties(envService.ApplicationEnvs.PropertyPath())
if err != nil {
return err
}
- dbClient, err = datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), envService.ApplicationEnvs.GoogleProjectId())
+ dbClient, err = datastore.New(ctx, mapper.NewPrecompiledObjectMapper(), externalFunctions, envService.ApplicationEnvs.GoogleProjectId())
if err != nil {
return err
}
downloadCatalogsToDatastoreEmulator(ctx)
- if err = setupDBStructure(ctx, dbClient, &envService.ApplicationEnvs, props); err != nil {
+ migrationVersion, err := dbClient.GetCurrentDbMigrationVersion(ctx)
+ if err != nil {
return err
}
+ envService.ApplicationEnvs.SetSchemaVersion(migrationVersion)
+
sdks, err := setupSdkCatalog(ctx, cacheService, dbClient)
if err != nil {
return err
@@ -97,7 +100,7 @@ func runServer() error {
// Since only router server has the scheduled task, the task creation is here
scheduledTasks := tasks.New(ctx)
- if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, props.RemovingUnusedSnptsDays, dbClient); err != nil {
+ if err = scheduledTasks.StartRemovingExtraSnippets(props.RemovingUnusedSnptsCron, externalFunctions); err != nil {
return err
}
}
@@ -224,26 +227,6 @@ func setupExamplesCatalogFromDatastore(ctx context.Context, cacheService cache.C
return nil
}
-// setupDBStructure initializes the data structure
-func setupDBStructure(ctx context.Context, db db.Database, appEnv *environment.ApplicationEnvs, props *environment.Properties) error {
- versions := []schema.Version{
- new(migration.InitialStructure),
- new(migration.AddingComplexityProperty),
- }
- dbSchema := schema.New(ctx, db, appEnv, props, versions)
- actualSchemaVersion, err := dbSchema.InitiateData()
- if err != nil {
- return err
- }
- if actualSchemaVersion == "" {
- errMsg := "schema version must not be empty"
- logger.Error(errMsg)
- return fmt.Errorf(errMsg)
- }
- appEnv.SetSchemaVersion(actualSchemaVersion)
- return nil
-}
-
func main() {
err := runServer()
if err != nil {
diff --git a/playground/backend/containers/router/Dockerfile b/playground/backend/containers/router/Dockerfile
index 75d123189d0..2aab1d579fd 100644
--- a/playground/backend/containers/router/Dockerfile
+++ b/playground/backend/containers/router/Dockerfile
@@ -39,6 +39,10 @@ RUN go mod tidy -v &&\
cd cmd/server &&\
go build -ldflags="-X main.BuildCommitHash=$GIT_COMMIT -X main.BuildCommitTimestamp=$GIT_TIMESTAMP" -o /go/bin/server_go_backend
+# Build migration tool
+RUN cd cmd/migration_tool &&\
+ go build -o /go/bin/migration_tool
+
# Null image
FROM debian:stable-20221114-slim
@@ -61,6 +65,7 @@ ENV PROPERTY_PATH=/opt/playground/backend/
# Copy build result
COPY --from=build /go/bin/server_go_backend /opt/playground/backend/
+COPY --from=build /go/bin/migration_tool /opt/playground/backend/
COPY --from=build /go/src/playground/backend/configs /opt/playground/backend/configs/
COPY sdks.yaml /opt/playground/backend/sdks.yaml
diff --git a/playground/backend/containers/router/docker-compose.local.yml b/playground/backend/containers/router/docker-compose.local.yml
index 2c48784d049..01c7732b8db 100644
--- a/playground/backend/containers/router/docker-compose.local.yml
+++ b/playground/backend/containers/router/docker-compose.local.yml
@@ -35,6 +35,7 @@ services:
CACHE_TYPE: remote
CACHE_ADDRESS: redis:6379
SDK_CONFIG: /opt/playground/backend/sdks-emulator.yaml
+ APPLY_MIGRATIONS: "True"
ports:
- "8080:8080"
depends_on:
diff --git a/playground/backend/containers/router/entrypoint.sh b/playground/backend/containers/router/entrypoint.sh
index 988302c2a71..ff19c84defa 100755
--- a/playground/backend/containers/router/entrypoint.sh
+++ b/playground/backend/containers/router/entrypoint.sh
@@ -14,4 +14,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Check if APPLY_MIGRATIONS is set and apply mgirations in such case
+if [ -n "$APPLY_MIGRATIONS" ]; then
+ echo "Applying migrations"
+ # If SDK_CONFIG is not set, set it to default value
+ if [ -z "$SDK_CONFIG" ]; then
+ SDK_CONFIG=/opt/playground/backend/sdks.yaml
+ fi
+
+ if [ -n "$DATASTORE_NAMESPACE" ]; then
+ /opt/playground/backend/migration_tool -project-id $GOOGLE_CLOUD_PROJECT -sdk-config $SDK_CONFIG -namespace $DATASTORE_NAMESPACE
+ else
+ /opt/playground/backend/migration_tool -project-id $GOOGLE_CLOUD_PROJECT -sdk-config $SDK_CONFIG
+ fi
+fi
+
/opt/playground/backend/server_go_backend
diff --git a/playground/backend/functions.go b/playground/backend/functions.go
new file mode 100644
index 00000000000..2d7b2c0ee40
--- /dev/null
+++ b/playground/backend/functions.go
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backend
+
+import (
+ "beam.apache.org/playground/backend/internal/constants"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "time"
+
+ "beam.apache.org/playground/backend/internal/db/datastore"
+ "beam.apache.org/playground/backend/internal/db/entity"
+ "beam.apache.org/playground/backend/internal/logger"
+ "beam.apache.org/playground/backend/playground_functions"
+
+ "beam.apache.org/playground/backend/internal/db/mapper"
+ "github.com/GoogleCloudPlatform/functions-framework-go/functions"
+)
+
+const retentionPeriod = 100 * time.Hour * 24
+
+var db *datastore.Datastore
+
+func init() {
+ env := playground_functions.GetEnvironment()
+ logger.SetupLogger(context.Background(), "", env.GetProjectId())
+
+ logger.Debugf("Initializing snippets functions\n")
+
+ pcMapper := mapper.NewPrecompiledObjectMapper()
+ var err error
+ db, err = datastore.New(context.Background(), pcMapper, nil, env.GetProjectId())
+ if err != nil {
+ fmt.Printf("Couldn't create the database client, err: %s\n", err.Error())
+ panic(err)
+ }
+
+ ensurePost := playground_functions.EnsureMethod(http.MethodPost)
+
+ functions.HTTP("cleanupSnippets", ensurePost(cleanupSnippets))
+ functions.HTTP("putSnippet", ensurePost(putSnippet))
+ functions.HTTP("incrementSnippetViews", ensurePost(incrementSnippetViews))
+}
+
+func handleError(w http.ResponseWriter, statusCode int, err error) {
+ // Return 500 error and error message
+ w.WriteHeader(statusCode)
+ _, werr := w.Write([]byte(err.Error()))
+ if werr != nil {
+ logger.Errorf("Couldn't write error message, err: %s \n", werr.Error())
+ }
+}
+
+// cleanupSnippets removes old snippets from the database.
+func cleanupSnippets(w http.ResponseWriter, r *http.Request) {
+ namespace := r.URL.Query().Get("namespace")
+ ctx := context.WithValue(r.Context(), constants.DatastoreNamespaceKey, namespace)
+
+ err := db.DeleteUnusedSnippets(ctx, retentionPeriod)
+ if err != nil {
+ logger.Errorf("Couldn't delete unused code snippets, err: %s \n", err.Error())
+ handleError(w, http.StatusInternalServerError, err)
+ return
+ }
+
+ w.WriteHeader(http.StatusOK)
+}
+
+func putSnippet(w http.ResponseWriter, r *http.Request) {
+ snipId := r.URL.Query().Get("snipId")
+ namespace := r.URL.Query().Get("namespace")
+ ctx := context.WithValue(r.Context(), constants.DatastoreNamespaceKey, namespace)
+
+ var snip entity.Snippet
+ err := json.NewDecoder(r.Body).Decode(&snip)
+ if err != nil {
+ logger.Errorf("Couldn't decode request body, err: %s \n", err.Error())
+ handleError(w, http.StatusBadRequest, err)
+ return
+ }
+
+ err = db.PutSnippetDirect(ctx, snipId, &snip)
+ if err != nil {
+ logger.Errorf("Couldn't put snippet to the database, err: %s \n", err.Error())
+ handleError(w, http.StatusInternalServerError, err)
+ return
+ }
+
+ w.WriteHeader(http.StatusOK)
+}
+
+func incrementSnippetViews(w http.ResponseWriter, r *http.Request) {
+ snipId := r.URL.Query().Get("snipId")
+ namespace := r.URL.Query().Get("namespace")
+ ctx := context.WithValue(r.Context(), constants.DatastoreNamespaceKey, namespace)
+
+ err := db.IncrementSnippetVisitorsCount(ctx, snipId)
+ if err != nil {
+ logger.Errorf("Couldn't increment snippet visitors count for snipId %s, err: %s \n", snipId, err.Error())
+ handleError(w, http.StatusInternalServerError, err)
+ return
+ }
+}
diff --git a/playground/backend/go.mod b/playground/backend/go.mod
index a2580231ff1..d1ba16a2d59 100644
--- a/playground/backend/go.mod
+++ b/playground/backend/go.mod
@@ -20,9 +20,11 @@ go 1.18
require (
cloud.google.com/go/datastore v1.9.0
cloud.google.com/go/logging v1.5.0
+ github.com/GoogleCloudPlatform/functions-framework-go v1.6.1
github.com/confluentinc/confluent-kafka-go v1.9.2
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redis/redismock/v8 v8.0.6
+ github.com/google/go-cmp v0.5.9
github.com/google/uuid v1.3.0
github.com/improbable-eng/grpc-web v0.15.0
github.com/linkedin/goavro v2.1.0+incompatible
@@ -30,7 +32,6 @@ require (
github.com/rs/cors v1.8.2
github.com/spf13/viper v1.14.0
github.com/stretchr/testify v1.8.1
- github.com/google/go-cmp v0.5.9
go.uber.org/goleak v1.2.0
google.golang.org/grpc v1.51.0
google.golang.org/protobuf v1.28.1
@@ -41,8 +42,10 @@ require (
cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/compute v1.12.1 // indirect
cloud.google.com/go/compute/metadata v0.2.1 // indirect
+ cloud.google.com/go/functions v1.7.0 // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
+ github.com/cloudevents/sdk-go/v2 v2.6.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -53,9 +56,12 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
github.com/googleapis/gax-go/v2 v2.6.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.11.7 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -65,6 +71,9 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
go.opencensus.io v0.23.0 // indirect
+ go.uber.org/atomic v1.9.0 // indirect
+ go.uber.org/multierr v1.8.0 // indirect
+ go.uber.org/zap v1.21.0 // indirect
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
golang.org/x/sync v0.1.0 // indirect
diff --git a/playground/backend/go.sum b/playground/backend/go.sum
index 09c9b06f440..1223bc87476 100644
--- a/playground/backend/go.sum
+++ b/playground/backend/go.sum
@@ -53,6 +53,9 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/datastore v1.9.0 h1:s3Gy1QRIwKxcMCCwJJq/4c64VjROZu6tq1DC632hZuo=
cloud.google.com/go/datastore v1.9.0/go.mod h1:yKk5PbPPCtuObGXNWvpQGEyWe+kiMQlTnpMjtltPNTc=
+cloud.google.com/go/functions v1.0.0/go.mod h1:O9KS8UweFVo6GbbbCBKh5yEzbW08PVkg2spe3RfPMd4=
+cloud.google.com/go/functions v1.7.0 h1:s3Snbr2O4j4p7CuwImBas8rNNmkHS1YJANcCpKGqQSE=
+cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg=
cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
cloud.google.com/go/logging v1.5.0 h1:DcR52smaYLgeK9KPzJlBJyyBYqW/EGKiuRRl8boL1s4=
cloud.google.com/go/logging v1.5.0/go.mod h1:c/57U/aLdzSFuBtvbtFduG1Ii54uSm95HOBnp58P7/U=
@@ -70,6 +73,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/GoogleCloudPlatform/functions-framework-go v1.6.1 h1:xy2RD54qi/vya4c+Jrh/3yS5JLcTpK167AY47AI4Tdc=
+github.com/GoogleCloudPlatform/functions-framework-go v1.6.1/go.mod h1:pq+lZy4vONJ5fjd3q/B6QzWhfHPAbuVweLpxZzMOb9Y=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
@@ -94,6 +99,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -112,6 +119,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudevents/sdk-go/v2 v2.6.1 h1:yHtzgmeBvc0TZx1nrnvYXov1CSvkQyvhEhNMs8Z5Mmk=
+github.com/cloudevents/sdk-go/v2 v2.6.1/go.mod h1:nlXhgFkf0uTopxmRXalyMwS2LG70cRGPrxzmjJgSG0U=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -286,6 +295,7 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -365,6 +375,7 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -423,6 +434,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -434,6 +446,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -472,6 +486,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
@@ -568,6 +583,8 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -594,13 +611,22 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -1024,6 +1050,7 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
@@ -1110,8 +1137,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
diff --git a/playground/backend/internal/db/datastore/datastore_db.go b/playground/backend/internal/db/datastore/datastore_db.go
index 6f4cbf074e9..c110a61487a 100644
--- a/playground/backend/internal/db/datastore/datastore_db.go
+++ b/playground/backend/internal/db/datastore/datastore_db.go
@@ -16,6 +16,7 @@
package datastore
import (
+ "beam.apache.org/playground/backend/internal/external_functions"
"context"
"fmt"
"time"
@@ -39,21 +40,26 @@ const (
)
type Datastore struct {
- Client *datastore.Client
- ResponseMapper mapper.ResponseMapper
+ Client *datastore.Client
+ ResponseMapper mapper.ResponseMapper
+ externalFunctions external_functions.ExternalFunctions
}
-func New(ctx context.Context, responseMapper mapper.ResponseMapper, projectId string) (*Datastore, error) {
+func New(ctx context.Context, responseMapper mapper.ResponseMapper, externalFunctions external_functions.ExternalFunctions, projectId string) (*Datastore, error) {
client, err := datastore.NewClient(ctx, projectId)
if err != nil {
logger.Errorf("Datastore: connection to store: error during connection, err: %s\n", err.Error())
return nil, err
}
- return &Datastore{Client: client, ResponseMapper: responseMapper}, nil
+ return &Datastore{
+ Client: client,
+ ResponseMapper: responseMapper,
+ externalFunctions: externalFunctions,
+ }, nil
}
// Delete unused snippets by given persistenceKey
-func (d *Datastore) deleteObsoleteSnippets(ctx context.Context, snipKey *datastore.Key, persistenceKey string) error {
+func (d *Datastore) DeleteObsoleteSnippets(ctx context.Context, snipKey *datastore.Key, persistenceKey string) error {
if persistenceKey == "" || snipKey == nil {
logger.Debugf("no persistence key or no current snippet key")
return nil
@@ -63,19 +69,42 @@ func (d *Datastore) deleteObsoleteSnippets(ctx context.Context, snipKey *datasto
Namespace(utils.GetNamespace(ctx)).
FilterField("persistenceKey", "=", persistenceKey)
- // At the moment, datastore emulator doesn't allow != filters,
- // hence this crutches
- // https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
- // When it's fixed, post-query filter could be replaced with
- //
- // FilterField("__key__", "!=", snipKey)
+ // At the moment, datastore emulator doesn't allow != filters,
+ // hence this crutches
+ // https://cloud.google.com/datastore/docs/tools/datastore-emulator#known_issues
+ // When it's fixed, post-query filter could be replaced with
+ //
+ // FilterField("__key__", "!=", snipKey)
return d.deleteSnippets(ctx, snippetQuery, snipKey)
}
-// PutSnippet puts the snippet entity to datastore
+// PutSnippet puts the snippet entity to datastore using cloud function proxy
func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip *entity.Snippet) error {
logger.Debugf("putting snippet %q, persistent key %q...", snipId, snip.Snippet.PersistenceKey)
+
+ var err error
+ if d.externalFunctions != nil {
+ err = d.externalFunctions.PutSnippet(ctx, snipId, snip)
+ }
+ if err != nil || d.externalFunctions == nil {
+ if err != nil {
+ logger.Errorf("Datastore: PutSnippet(): error during the PutSnippet() call to the cloud function, "+
+ "accessing the datastore directly, err: %s\n", err.Error())
+ }
+ if d.externalFunctions == nil {
+ logger.Warnf("Datastore: PutSnippet(): external functions are not set, " +
+ "accessing the datastore directly")
+ }
+ return d.PutSnippetDirect(ctx, snipId, snip)
+ }
+
+ return nil
+}
+
+// PutSnippetDirect puts the snippet entity to datastore
+func (d *Datastore) PutSnippetDirect(ctx context.Context, snipId string, snip *entity.Snippet) error {
+ logger.Debugf("putting snippet %q, persistent key %q...", snipId, snip.Snippet.PersistenceKey)
if snip == nil {
logger.Errorf("Datastore: PutSnippet(): snippet is nil")
return nil
@@ -105,7 +134,8 @@ func (d *Datastore) PutSnippet(ctx context.Context, snipId string, snip *entity.
return err
}
- return d.deleteObsoleteSnippets(ctx, snipKey, snip.Snippet.PersistenceKey)
+ // Delete the previous version of the snippet
+ return d.DeleteObsoleteSnippets(ctx, snipKey, snip.Snippet.PersistenceKey)
}
// GetSnippet returns the snippet entity by identifier
@@ -113,53 +143,56 @@ func (d *Datastore) GetSnippet(ctx context.Context, id string) (*entity.SnippetE
key := utils.GetSnippetKey(ctx, id)
snip := new(entity.SnippetEntity)
+ err := d.Client.Get(ctx, key, snip)
+ if err != nil {
+ logger.Errorf("Datastore: GetSnippet(): error during snippet getting, err: %s\n", err.Error())
+ return nil, err
+ }
+
+ logger.Infof("Datastore: GetSnippet(): snippet %s has %d view count", id, snip.VisitCount)
+
+ // Update the last visited time and visit count if possible
+ err = nil
+ if d.externalFunctions != nil {
+ err = d.externalFunctions.IncrementSnippetViews(ctx, id)
+ }
+ if err != nil || d.externalFunctions == nil {
+ if err != nil {
+ logger.Errorf("Datastore: GetSnippet(): error during updating snippet visit count using"+
+ " cloud function proxy, err: %s\n", err.Error())
+ }
+ if d.externalFunctions == nil {
+ logger.Warnf("Datastore: GetSnippet(): cloud function proxy is not initialized, " +
+ "trying to call IncrementSnippetVisitorsCount() directly.")
+ }
+ err = d.IncrementSnippetVisitorsCount(ctx, id)
+ if err != nil {
+ logger.Errorf("Datastore: GetSnippet(): Can't increment snippet visit count, skipping view increment")
+ }
+ }
+
+ return snip, nil
+}
+
+func (d *Datastore) IncrementSnippetVisitorsCount(ctx context.Context, id string) error {
+ key := utils.GetSnippetKey(ctx, id)
+ snip := new(entity.SnippetEntity)
+
_, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
if err := tx.Get(key, snip); err != nil {
- logger.Errorf("Datastore: GetSnippet(): error during snippet getting, err: %s\n", err.Error())
+ logger.Errorf("Datastore: IncrementSnippetVisitorsCount(): error during snippet getting, err: %s\n", err.Error())
return err
}
snip.LVisited = time.Now()
snip.VisitCount += 1
if _, err := tx.Put(key, snip); err != nil {
- logger.Errorf("Datastore: GetSnippet(): error during snippet setting, err: %s\n", err.Error())
+ logger.Errorf("Datastore: IncrementSnippetVisitorsCount(): error during snippet setting, err: %s\n", err.Error())
return err
}
return nil
})
if err != nil {
- logger.Errorf("Datastore: GetSnippet: error updating snippet: %s\n", err.Error())
- return nil, err
- }
-
- return snip, nil
-}
-
-// PutSchemaVersion puts the schema entity to datastore
-func (d *Datastore) PutSchemaVersion(ctx context.Context, id string, schema *entity.SchemaEntity) error {
- if schema == nil {
- logger.Errorf("Datastore: PutSchemaVersion(): schema version is nil")
- return nil
- }
- key := utils.GetSchemaVerKey(ctx, id)
- if _, err := d.Client.Put(ctx, key, schema); err != nil {
- logger.Errorf("Datastore: PutSchemaVersion(): error during entity saving, err: %s\n", err.Error())
- return err
- }
- return nil
-}
-
-// PutSDKs puts the SDK entity to datastore
-func (d *Datastore) PutSDKs(ctx context.Context, sdks []*entity.SDKEntity) error {
- if sdks == nil || len(sdks) == 0 {
- logger.Errorf("Datastore: PutSDKs(): sdks are empty")
- return nil
- }
- var keys []*datastore.Key
- for _, sdk := range sdks {
- keys = append(keys, utils.GetSdkKey(ctx, sdk.Name))
- }
- if _, err := d.Client.PutMulti(ctx, keys, sdks); err != nil {
- logger.Errorf("Datastore: PutSDK(): error during entity saving, err: %s\n", err.Error())
+ logger.Errorf("Datastore: IncrementSnippetVisitorsCount: error updating snippet: %s\n", err.Error())
return err
}
return nil
@@ -536,10 +569,9 @@ func (d *Datastore) DeleteSnippet(ctx context.Context, id string) error {
return d.deleteSnippetByKey(ctx, key)
}
-// DeleteUnusedSnippets deletes all unused snippets
-func (d *Datastore) DeleteUnusedSnippets(ctx context.Context, dayDiff int32) error {
- var hoursDiff = dayDiff * 24
- boundaryDate := time.Now().Add(-time.Hour * time.Duration(hoursDiff))
+// DeleteUnusedSnippets deletes all unused snippets older than retentionPeriod
+func (d *Datastore) DeleteUnusedSnippets(ctx context.Context, retentionPeriod time.Duration) error {
+ boundaryDate := time.Now().Add(-retentionPeriod)
snippetQuery := datastore.NewQuery(constants.SnippetKind).
Namespace(utils.GetNamespace(ctx)).
FilterField("lVisited", "<=", boundaryDate).
diff --git a/playground/backend/internal/db/datastore/datastore_db_test.go b/playground/backend/internal/db/datastore/datastore_db_test.go
index 835ee90e163..61d8e35049c 100644
--- a/playground/backend/internal/db/datastore/datastore_db_test.go
+++ b/playground/backend/internal/db/datastore/datastore_db_test.go
@@ -249,53 +249,6 @@ func TestDatastore_PutSDKs(t *testing.T) {
}
}
-func TestDatastore_PutSchemaVersion(t *testing.T) {
- type args struct {
- ctx context.Context
- id string
- schema *entity.SchemaEntity
- }
- tests := []struct {
- name string
- args args
- wantErr bool
- cleanData func()
- }{
- {
- name: "PutSchemaVersion() in the usual case",
- args: args{
- ctx: ctx,
- id: "MOCK_ID",
- schema: &entity.SchemaEntity{Descr: "MOCK_DESCRIPTION"},
- },
- wantErr: false,
- cleanData: func() {
- test_cleaner.CleanSchemaVersion(ctx, t, "MOCK_ID")
- },
- },
- {
- name: "PutSchemaVersion() when input data is nil",
- args: args{
- ctx: ctx,
- id: "MOCK_ID",
- schema: nil,
- },
- wantErr: false,
- cleanData: func() {},
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- err := datastoreDb.PutSchemaVersion(tt.args.ctx, tt.args.id, tt.args.schema)
- if err != nil {
- t.Error("PutSchemaVersion() method failed")
- }
- tt.cleanData()
- })
- }
-}
-
func TestDatastore_GetFiles(t *testing.T) {
type args struct {
ctx context.Context
@@ -858,7 +811,7 @@ func TestDatastore_DeleteUnusedSnippets(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.prepare()
- err := datastoreDb.DeleteUnusedSnippets(tt.args.ctx, tt.args.dayDiff)
+ err := datastoreDb.DeleteUnusedSnippets(tt.args.ctx, time.Duration(tt.args.dayDiff)*time.Hour*24)
if (err != nil) != tt.wantErr {
t.Errorf("DeleteUnusedSnippets() error = %v, wantErr %v", err, tt.wantErr)
}
@@ -939,7 +892,7 @@ func TestNew(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- _, err := New(ctx, mapper.NewPrecompiledObjectMapper(), constants.EmulatorProjectId)
+ _, err := New(ctx, mapper.NewPrecompiledObjectMapper(), nil, constants.EmulatorProjectId)
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
}
diff --git a/playground/backend/internal/db/datastore/emulator_wrapper.go b/playground/backend/internal/db/datastore/emulator_wrapper.go
index 2aff281f2a0..9252c454b01 100644
--- a/playground/backend/internal/db/datastore/emulator_wrapper.go
+++ b/playground/backend/internal/db/datastore/emulator_wrapper.go
@@ -81,7 +81,7 @@ func NewEmulated(ctx context.Context) (*EmulatedDatastore, error) {
panic(err)
}
- datastoreDb, err := New(ctx, mapper.NewPrecompiledObjectMapper(), constants.EmulatorProjectId)
+ datastoreDb, err := New(ctx, mapper.NewPrecompiledObjectMapper(), nil, constants.EmulatorProjectId)
if err != nil {
return nil, err
}
diff --git a/playground/backend/internal/db/schema/migration/migration_v002.go b/playground/backend/internal/db/datastore/migration_base.go
similarity index 56%
rename from playground/backend/internal/db/schema/migration/migration_v002.go
rename to playground/backend/internal/db/datastore/migration_base.go
index 4d2ed04c7ae..f58b0a5d2d2 100644
--- a/playground/backend/internal/db/schema/migration/migration_v002.go
+++ b/playground/backend/internal/db/datastore/migration_base.go
@@ -13,28 +13,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package migration
+package datastore
import (
- "beam.apache.org/playground/backend/internal/db/entity"
- "beam.apache.org/playground/backend/internal/db/schema"
+ "cloud.google.com/go/datastore"
+ "context"
)
-type AddingComplexityProperty struct {
+type Migration interface {
+ Apply(ctx context.Context, tx *datastore.Transaction, sdkConfigPath string) error
+ GetVersion() int
+ GetDescription() string
}
-func (is *AddingComplexityProperty) InitiateData(args *schema.DBArgs) error {
- schemaEntity := &entity.SchemaEntity{Descr: is.GetDescription()}
- if err := args.Db.PutSchemaVersion(args.Ctx, is.GetVersion(), schemaEntity); err != nil {
- return err
- }
- return nil
+type MigrationBase struct {
+ Version int
+ Description string
}
-func (is *AddingComplexityProperty) GetVersion() string {
- return "0.0.2"
+func (m MigrationBase) GetVersion() int {
+ return m.Version
}
-func (is AddingComplexityProperty) GetDescription() string {
- return "Adding a complexity property to the example entity"
+func (m MigrationBase) GetDescription() string {
+ return m.Description
}
diff --git a/playground/backend/internal/db/datastore/migration_db.go b/playground/backend/internal/db/datastore/migration_db.go
new file mode 100644
index 00000000000..72946bedc0c
--- /dev/null
+++ b/playground/backend/internal/db/datastore/migration_db.go
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datastore
+
+import (
+ "beam.apache.org/playground/backend/internal/constants"
+ "beam.apache.org/playground/backend/internal/db/entity"
+ "beam.apache.org/playground/backend/internal/logger"
+ "beam.apache.org/playground/backend/internal/utils"
+ "cloud.google.com/go/datastore"
+ "context"
+ "errors"
+)
+
+// GetCurrentDbMigrationVersion returns the current version of the schema
+func (d *Datastore) GetCurrentDbMigrationVersion(ctx context.Context) (int, error) {
+ query := datastore.NewQuery(constants.SchemaKind).
+ Namespace(utils.GetNamespace(ctx)).Order("-version").Limit(1)
+ var schemas []*entity.SchemaEntity
+ if _, err := d.Client.GetAll(ctx, query, &schemas); err != nil {
+ logger.Errorf("Datastore: GetCurrentDbMigrationVersion(): error during getting current version, err: %s\n", err.Error())
+ return -1, err
+ }
+ if len(schemas) == 0 {
+ logger.Errorf("Datastore: GetCurrentDbMigrationVersion(): no schema versions found\n")
+ return -1, errors.New("no schema versions found")
+ }
+ return schemas[0].Version, nil
+}
+
+// HasSchemaVersion returns true if the schema version is applied
+func (d *Datastore) hasSchemaVersion(ctx context.Context, version int) (bool, error) {
+ key := utils.GetSchemaVerKey(ctx, version)
+ schemaEntity := new(entity.SchemaEntity)
+ err := d.Client.Get(ctx, key, schemaEntity)
+ if err != nil {
+ if err == datastore.ErrNoSuchEntity {
+ return false, nil
+ }
+ logger.Errorf("Datastore: hasSchemaVersion(): error during getting schema version, err: %s\n", err.Error())
+ return false, err
+ }
+ logger.Infof("Datastore: hasSchemaVersion(): found SchemaEntity: %v\n", schemaEntity)
+ return true, nil
+}
+
+// applyMigration applies the given migration to the database.
+func (d *Datastore) applyMigration(ctx context.Context, migration Migration, sdkConfigPath string) error {
+ logger.Infof("Datastore: applyMigration(): applying migration \"%d: %s\"\n", migration.GetVersion(), migration.GetDescription())
+
+ _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
+ if err := migration.Apply(ctx, tx, sdkConfigPath); err != nil {
+ logger.Errorf("Datastore: applyMigration(): error during migration \"%d: %s\" applying, rolling back, err: %s\n",
+ migration.GetVersion(),
+ migration.GetDescription(),
+ err.Error())
+ return err
+ }
+
+ // Record the migration version
+ if err := putSchemaVersion(ctx, tx, migration.GetVersion(), migration.GetDescription()); err != nil {
+ logger.Errorf("Datastore: applyMigration(): error during migration \"%d: %s\" applying, rolling back, err: %s\n",
+ migration.GetVersion(),
+ migration.GetDescription(),
+ err.Error())
+ return err
+ }
+
+ return nil
+ })
+
+ if err != nil {
+ logger.Errorf("Datastore: applyMigration(): error during migration \"%d: %s\" applying, err: %s\n",
+ migration.GetVersion(),
+ migration.GetDescription(),
+ err.Error())
+ return err
+ }
+
+ logger.Infof("Datastore: applyMigration(): migration \"%d: %s\" applied successfully\n", migration.GetVersion(), migration.GetDescription())
+ return nil
+}
+
+// ApplyMigrations applies all migrations to the database.
+func (d *Datastore) ApplyMigrations(ctx context.Context, migrations []Migration, sdkConfigPath string) error {
+ for _, migration := range migrations {
+ if applied, err := d.hasSchemaVersion(ctx, migration.GetVersion()); err != nil {
+ logger.Errorf("Datastore: ApplyMigrations(): Error checking migration \"%d: %s\" : %s", migration.GetVersion(), migration.GetDescription(), err.Error())
+ return err
+ } else if applied {
+ logger.Infof("Datastore: ApplyMigrations(): migration \"%d: %s\" already applied, skipping\n", migration.GetVersion(), migration.GetDescription())
+ continue
+ }
+
+ if err := d.applyMigration(ctx, migration, sdkConfigPath); err != nil {
+ logger.Errorf("Datastore: ApplyMigrations(): Error applying migration \"%d: %s\" : %s", migration.GetVersion(), migration.GetDescription(), err.Error())
+ return err
+ }
+ }
+ return nil
+}
+
+// putSchemaVersion puts the schema entity to datastore
+func putSchemaVersion(ctx context.Context, tx *datastore.Transaction, version int, description string) error {
+ key := utils.GetSchemaVerKey(ctx, version)
+ if _, err := tx.Put(key, &entity.SchemaEntity{
+ Version: version,
+ Descr: description,
+ }); err != nil {
+ logger.Errorf("Datastore: putSchemaVersion(): error during entity saving, err: %s\n", err.Error())
+ return err
+ }
+ return nil
+}
+
+func (d *Datastore) PutSDKs(ctx context.Context, sdks []*entity.SDKEntity) error {
+ _, err := d.Client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
+ return TxPutSDKs(ctx, tx, sdks)
+ })
+
+ if err != nil {
+ logger.Errorf("Datastore: PutSDKs(): error during transaction, err: %s\n", err.Error())
+ return err
+ }
+
+ return nil
+}
+
+// TxPutSDKs puts the SDK entity to datastore in a transaction
+func TxPutSDKs(ctx context.Context, tx *datastore.Transaction, sdks []*entity.SDKEntity) error {
+ if sdks == nil || len(sdks) == 0 {
+ logger.Errorf("Datastore: TxPutSDKs(): sdks are empty")
+ return nil
+ }
+ var keys []*datastore.Key
+ for _, sdk := range sdks {
+ keys = append(keys, utils.GetSdkKey(ctx, sdk.Name))
+ }
+ if _, err := tx.PutMulti(keys, sdks); err != nil {
+ logger.Errorf("Datastore: TxPutSDKs(): error during entity saving, err: %s\n", err.Error())
+ return err
+ }
+ return nil
+}
diff --git a/playground/backend/internal/db/db.go b/playground/backend/internal/db/db.go
index 8f73a57a59a..ff1cb5154e8 100644
--- a/playground/backend/internal/db/db.go
+++ b/playground/backend/internal/db/db.go
@@ -16,7 +16,9 @@
package db
import (
+ "beam.apache.org/playground/backend/internal/db/datastore"
"context"
+ "time"
pb "beam.apache.org/playground/backend/internal/api/v1"
"beam.apache.org/playground/backend/internal/db/entity"
@@ -26,6 +28,7 @@ type Database interface {
SnippetDatabase
CatalogDatabase
ExampleDatabase
+ MigrationsDatabase
}
type SnippetDatabase interface {
@@ -35,14 +38,10 @@ type SnippetDatabase interface {
GetFiles(ctx context.Context, snipId string, numberOfFiles int) ([]*entity.FileEntity, error)
- DeleteUnusedSnippets(ctx context.Context, dayDiff int32) error
+ DeleteUnusedSnippets(ctx context.Context, retentionPeriod time.Duration) error
}
type CatalogDatabase interface {
- PutSchemaVersion(ctx context.Context, id string, schema *entity.SchemaEntity) error
-
- PutSDKs(ctx context.Context, sdks []*entity.SDKEntity) error
-
GetSDKs(ctx context.Context) ([]*entity.SDKEntity, error)
}
@@ -61,3 +60,9 @@ type ExampleDatabase interface {
GetExampleGraph(ctx context.Context, id string) (string, error)
}
+
+type MigrationsDatabase interface {
+ GetCurrentDbMigrationVersion(ctx context.Context) (int, error)
+
+ ApplyMigrations(ctx context.Context, migrations []datastore.Migration, sdkConfigPath string) error
+}
diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/internal/db/entity/schema.go
index 43aaacfca34..a0fd0e72e3e 100644
--- a/playground/backend/internal/db/entity/schema.go
+++ b/playground/backend/internal/db/entity/schema.go
@@ -16,5 +16,6 @@
package entity
type SchemaEntity struct {
- Descr string `datastore:"descr,noindex"`
+ Version int `datastore:"version"`
+ Descr string `datastore:"descr,noindex"`
}
diff --git a/playground/backend/internal/db/mapper/datastore_mapper_test.go b/playground/backend/internal/db/mapper/datastore_mapper_test.go
index b84694824d2..3acb44a9530 100644
--- a/playground/backend/internal/db/mapper/datastore_mapper_test.go
+++ b/playground/backend/internal/db/mapper/datastore_mapper_test.go
@@ -31,8 +31,8 @@ var testable *DatastoreMapper
var datastoreMapperCtx = context.Background()
func TestMain(m *testing.M) {
- appEnv := environment.NewApplicationEnvs("/app", "", "", "", "", "../../../.", "", "", nil, 0, 0)
- appEnv.SetSchemaVersion("MOCK_SCHEMA")
+ appEnv := environment.NewApplicationEnvs("/app", "", "", "", "", "../../../.", "", "", "", "", "", nil, 0, 0)
+ appEnv.SetSchemaVersion(1)
props, _ := environment.NewProperties(appEnv.PropertyPath())
testable = NewDatastoreMapper(datastoreMapperCtx, appEnv, props)
exitValue := m.Run()
@@ -59,7 +59,7 @@ func TestEntityMapper_ToSnippet(t *testing.T) {
IdLength: 11,
},
Snippet: &entity.SnippetEntity{
- SchVer: utils.GetSchemaVerKey(datastoreMapperCtx, "MOCK_SCHEMA"),
+ SchVer: utils.GetSchemaVerKey(datastoreMapperCtx, 1),
Sdk: utils.GetSdkKey(datastoreMapperCtx, pb.Sdk_SDK_JAVA.String()),
PipeOpts: "MOCK_OPTIONS",
Origin: constants.UserSnippetOrigin,
diff --git a/playground/backend/internal/db/schema/migration/migrations_test.go b/playground/backend/internal/db/schema/migration/migrations_test.go
deleted file mode 100644
index 42218e9f4ac..00000000000
--- a/playground/backend/internal/db/schema/migration/migrations_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package migration
-
-import (
- "context"
- "os"
- "testing"
-
- "beam.apache.org/playground/backend/internal/constants"
- "beam.apache.org/playground/backend/internal/db/datastore"
- "beam.apache.org/playground/backend/internal/db/schema"
- "beam.apache.org/playground/backend/internal/environment"
-)
-
-var datastoreDb *datastore.EmulatedDatastore
-var ctx context.Context
-var appEnvs *environment.ApplicationEnvs
-var props *environment.Properties
-
-func TestMain(m *testing.M) {
- setup()
- code := m.Run()
- teardown()
- os.Exit(code)
-}
-
-func setup() {
- ctx = context.Background()
- ctx = context.WithValue(ctx, constants.DatastoreNamespaceKey, "migration")
- var err error
- datastoreDb, err = datastore.NewEmulated(ctx)
- if err != nil {
- panic(err)
- }
- if err != nil {
- panic(err)
- }
- appEnvs = environment.NewApplicationEnvs("/app", "", "", "", "../../../../../sdks-emulator.yaml", "../../../../.", "", "", nil, 0, 0)
- props, err = environment.NewProperties(appEnvs.PropertyPath())
- if err != nil {
- panic(err)
- }
-}
-
-func teardown() {
- clientCloseErr := datastoreDb.Close()
- if clientCloseErr != nil {
- panic(clientCloseErr)
- }
-}
-
-func TestInitialStructure_InitiateData(t *testing.T) {
- tests := []struct {
- name string
- dbArgs *schema.DBArgs
- wantErr bool
- }{
- {
- name: "Test migration with version 0.0.1 in the usual case",
- dbArgs: &schema.DBArgs{
- Ctx: ctx,
- Db: datastoreDb,
- AppEnv: appEnvs,
- Props: props,
- },
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- is := new(InitialStructure)
- err := is.InitiateData(tt.dbArgs)
- if (err != nil) != tt.wantErr {
- t.Errorf("InitiateData(): error = %v, wantErr %v", err, tt.wantErr)
- }
- })
- }
-}
-
-func TestAddingComplexityProperty_InitiateData(t *testing.T) {
- tests := []struct {
- name string
- dbArgs *schema.DBArgs
- wantErr bool
- }{
- {
- name: "Test migration with version 0.0.2 in the usual case",
- dbArgs: &schema.DBArgs{
- Ctx: ctx,
- Db: datastoreDb,
- AppEnv: appEnvs,
- Props: props,
- },
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- is := new(AddingComplexityProperty)
- err := is.InitiateData(tt.dbArgs)
- if (err != nil) != tt.wantErr {
- t.Errorf("InitiateData(): error = %v, wantErr %v", err, tt.wantErr)
- }
- })
- }
-
-}
diff --git a/playground/backend/internal/db/schema/migration/migration_v001.go b/playground/backend/internal/db/schema/migration_v001.go
similarity index 74%
rename from playground/backend/internal/db/schema/migration/migration_v001.go
rename to playground/backend/internal/db/schema/migration_v001.go
index 38550859c65..f132451abcc 100644
--- a/playground/backend/internal/db/schema/migration/migration_v001.go
+++ b/playground/backend/internal/db/schema/migration_v001.go
@@ -13,29 +13,35 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package migration
+package schema
import (
pb "beam.apache.org/playground/backend/internal/api/v1"
+ ds "beam.apache.org/playground/backend/internal/db/datastore"
"beam.apache.org/playground/backend/internal/db/entity"
- "beam.apache.org/playground/backend/internal/db/schema"
"beam.apache.org/playground/backend/internal/utils"
+ "cloud.google.com/go/datastore"
+ "context"
)
-type InitialStructure struct {
+type migrationV001 struct {
+ ds.MigrationBase
}
-func (is *InitialStructure) InitiateData(args *schema.DBArgs) error {
- //init schema versions
- schemaEntity := &entity.SchemaEntity{Descr: is.GetDescription()}
- if err := args.Db.PutSchemaVersion(args.Ctx, is.GetVersion(), schemaEntity); err != nil {
- return err
+func GetMigration_v001() ds.Migration {
+ return &migrationV001{
+ MigrationBase: ds.MigrationBase{
+ Version: 1,
+ Description: "Data initialization: a schema version, SDKs",
+ },
}
+}
- //init sdks
+func (m migrationV001) Apply(ctx context.Context, tx *datastore.Transaction, sdkConfigPath string) error {
+ // Init sdks
var sdkEntities []*entity.SDKEntity
sdkConfig := new(SdkConfig)
- if err := utils.ReadYamlFile(args.AppEnv.SdkConfigPath(), sdkConfig); err != nil {
+ if err := utils.ReadYamlFile(sdkConfigPath, sdkConfig); err != nil {
return err
}
for _, sdk := range pb.Sdk_name {
@@ -48,7 +54,7 @@ func (is *InitialStructure) InitiateData(args *schema.DBArgs) error {
DefaultExample: defaultExample,
})
}
- if err := args.Db.PutSDKs(args.Ctx, sdkEntities); err != nil {
+ if err := ds.TxPutSDKs(ctx, tx, sdkEntities); err != nil {
return err
}
@@ -82,11 +88,3 @@ func getDefaultExample(config *SdkConfig, sdk string) string {
return ""
}
}
-
-func (is *InitialStructure) GetVersion() string {
- return "0.0.1"
-}
-
-func (is InitialStructure) GetDescription() string {
- return "Data initialization: a schema version, SDKs"
-}
diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/internal/db/schema/migrations.go
similarity index 81%
copy from playground/backend/internal/db/entity/schema.go
copy to playground/backend/internal/db/schema/migrations.go
index 43aaacfca34..690a57509a9 100644
--- a/playground/backend/internal/db/entity/schema.go
+++ b/playground/backend/internal/db/schema/migrations.go
@@ -13,8 +13,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package entity
+package schema
-type SchemaEntity struct {
- Descr string `datastore:"descr,noindex"`
+import ds "beam.apache.org/playground/backend/internal/db/datastore"
+
+// Migrations List of all migrations
+var Migrations = []ds.Migration{
+ GetMigration_v001(),
}
diff --git a/playground/backend/internal/db/schema/version.go b/playground/backend/internal/db/schema/version.go
deleted file mode 100644
index 457011aa9e9..00000000000
--- a/playground/backend/internal/db/schema/version.go
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package schema
-
-import (
- "beam.apache.org/playground/backend/internal/db"
- "beam.apache.org/playground/backend/internal/environment"
- "beam.apache.org/playground/backend/internal/logger"
- "context"
- "sort"
-)
-
-type DBArgs struct {
- Ctx context.Context
- Db db.Database
- AppEnv *environment.ApplicationEnvs
- Props *environment.Properties
-}
-
-type DBSchema struct {
- args *DBArgs
- versions []Version
-}
-
-func New(ctx context.Context, db db.Database, appEnv *environment.ApplicationEnvs, props *environment.Properties, versions []Version) *DBSchema {
- return &DBSchema{
- args: &DBArgs{ctx, db, appEnv, props},
- versions: versions,
- }
-}
-
-func (ds *DBSchema) InitiateData() (string, error) {
- var versions []string
- for _, ver := range ds.versions {
- if err := ver.InitiateData(ds.args); err != nil {
- logger.Errorf("DBSchema: InitiateData() error during the data initialization, err: %s", err.Error())
- return "", err
- }
- versions = append(versions, ver.GetVersion())
- }
- sort.Strings(versions)
- return versions[len(versions)-1], nil
-}
-
-type Version interface {
- GetVersion() string
- GetDescription() string
- InitiateData(args *DBArgs) error
-}
diff --git a/playground/backend/internal/environment/application.go b/playground/backend/internal/environment/application.go
index 1b3e1e37bb0..e61fcea5563 100644
--- a/playground/backend/internal/environment/application.go
+++ b/playground/backend/internal/environment/application.go
@@ -101,7 +101,7 @@ type ApplicationEnvs struct {
pipelinesFolder string
// schemaVersion is the database schema version
- schemaVersion string
+ schemaVersion int
// sdkConfigPath is a sdk configuration file
sdkConfigPath string
@@ -117,26 +117,48 @@ type ApplicationEnvs struct {
// cacheRequestTimeout is timeout to request data from cache
cacheRequestTimeout time.Duration
+
+ // cleanupSnippetsFunctionsUrl is the url to cleanup snippets functions
+ cleanupSnippetsFunctionsUrl string
+
+ // putSnippetFunctionUrl is the url to put snippet function
+ putSnippetFunctionsUrl string
+
+ // incrementSnippetViewsFunctionsUrl is the url to increment snippet views
+ incrementSnippetViewsFunctionsUrl string
}
// NewApplicationEnvs constructor for ApplicationEnvs
func NewApplicationEnvs(
- workingDir, launchSite, projectId, pipelinesFolder, sdkConfigPath, propertyPath, kafkaEmulatorExecutablePath, datasetsPath string,
+ workingDir,
+ launchSite,
+ projectId,
+ pipelinesFolder,
+ sdkConfigPath,
+ propertyPath,
+ kafkaEmulatorExecutablePath,
+ datasetsPath,
+ cleanupSnippetsFunctionsUrl,
+ putSnippetFunctionsUrl,
+ incrementSnippetViewsFunctionsUrl string,
cacheEnvs *CacheEnvs,
pipelineExecuteTimeout, cacheRequestTimeout time.Duration,
) *ApplicationEnvs {
return &ApplicationEnvs{
- workingDir: workingDir,
- cacheEnvs: cacheEnvs,
- pipelineExecuteTimeout: pipelineExecuteTimeout,
- launchSite: launchSite,
- projectId: projectId,
- pipelinesFolder: pipelinesFolder,
- sdkConfigPath: sdkConfigPath,
- propertyPath: propertyPath,
- datasetsPath: datasetsPath,
- kafkaEmulatorExecutablePath: kafkaEmulatorExecutablePath,
- cacheRequestTimeout: cacheRequestTimeout,
+ workingDir: workingDir,
+ cacheEnvs: cacheEnvs,
+ pipelineExecuteTimeout: pipelineExecuteTimeout,
+ launchSite: launchSite,
+ projectId: projectId,
+ pipelinesFolder: pipelinesFolder,
+ sdkConfigPath: sdkConfigPath,
+ propertyPath: propertyPath,
+ datasetsPath: datasetsPath,
+ kafkaEmulatorExecutablePath: kafkaEmulatorExecutablePath,
+ cacheRequestTimeout: cacheRequestTimeout,
+ cleanupSnippetsFunctionsUrl: cleanupSnippetsFunctionsUrl,
+ putSnippetFunctionsUrl: putSnippetFunctionsUrl,
+ incrementSnippetViewsFunctionsUrl: incrementSnippetViewsFunctionsUrl,
}
}
@@ -171,7 +193,7 @@ func (ae *ApplicationEnvs) PipelinesFolder() string {
}
// SchemaVersion returns the database schema version
-func (ae *ApplicationEnvs) SchemaVersion() string {
+func (ae *ApplicationEnvs) SchemaVersion() int {
return ae.schemaVersion
}
@@ -186,7 +208,7 @@ func (ae *ApplicationEnvs) PropertyPath() string {
}
// SetSchemaVersion sets the database schema version
-func (ae *ApplicationEnvs) SetSchemaVersion(schemaVersion string) {
+func (ae *ApplicationEnvs) SetSchemaVersion(schemaVersion int) {
ae.schemaVersion = schemaVersion
}
@@ -203,3 +225,18 @@ func (ae *ApplicationEnvs) DatasetsPath() string {
func (ae *ApplicationEnvs) KafkaExecutablePath() string {
return ae.kafkaEmulatorExecutablePath
}
+
+// CleanupSnippetsFunctionsUrl returns the url to cleanup snippets functions
+func (ae *ApplicationEnvs) CleanupSnippetsFunctionsUrl() string {
+ return ae.cleanupSnippetsFunctionsUrl
+}
+
+// PutSnippetFunctionUrl returns the url to put snippet functions
+func (ae *ApplicationEnvs) PutSnippetFunctionsUrl() string {
+ return ae.putSnippetFunctionsUrl
+}
+
+// IncrementSnippetViewsFunctionsUrl returns the url to increment snippet views
+func (ae *ApplicationEnvs) IncrementSnippetViewsFunctionsUrl() string {
+ return ae.incrementSnippetViewsFunctionsUrl
+}
diff --git a/playground/backend/internal/environment/environment_service.go b/playground/backend/internal/environment/environment_service.go
index 6536a4656a4..27e4e0d511a 100644
--- a/playground/backend/internal/environment/environment_service.go
+++ b/playground/backend/internal/environment/environment_service.go
@@ -32,47 +32,53 @@ import (
)
const (
- serverIpKey = "SERVER_IP"
- serverPortKey = "SERVER_PORT"
- beamSdkKey = "BEAM_SDK"
- beamVersionKey = "BEAM_VERSION"
- workingDirKey = "APP_WORK_DIR"
- preparedModDirKey = "PREPARED_MOD_DIR"
- numOfParallelJobsKey = "NUM_PARALLEL_JOBS"
- cacheTypeKey = "CACHE_TYPE"
- cacheAddressKey = "CACHE_ADDRESS"
- beamPathKey = "BEAM_PATH"
- cacheKeyExpirationTimeKey = "KEY_EXPIRATION_TIME"
- pipelineExecuteTimeoutKey = "PIPELINE_EXPIRATION_TIMEOUT"
- protocolTypeKey = "PROTOCOL_TYPE"
- launchSiteKey = "LAUNCH_SITE"
- projectIdKey = "GOOGLE_CLOUD_PROJECT"
- pipelinesFolderKey = "PIPELINES_FOLDER_NAME"
- defaultPipelinesFolder = "executable_files"
- defaultLaunchSite = "local"
- defaultProtocol = "HTTP"
- defaultIp = "localhost"
- defaultPort = 8080
- defaultSdk = pb.Sdk_SDK_UNSPECIFIED
- defaultBeamVersion = "<unknown>"
- defaultBeamJarsPath = "/opt/apache/beam/jars/*"
- defaultDatasetsPath = "/opt/playground/backend/datasets"
- defaultKafkaEmulatorExecutablePath = "/opt/playground/backend/kafka-emulator/beam-playground-kafka-emulator.jar"
- defaultCacheType = "local"
- defaultCacheAddress = "localhost:6379"
- defaultCacheKeyExpirationTime = time.Minute * 15
- defaultPipelineExecuteTimeout = time.Minute * 10
- jsonExt = ".json"
- configFolderName = "configs"
- defaultNumOfParallelJobs = 20
- SDKConfigPathKey = "SDK_CONFIG"
- defaultSDKConfigPath = "../sdks.yaml"
- propertyPathKey = "PROPERTY_PATH"
- datasetsPathKey = "DATASETS_PATH"
- kafkaEmulatorExecutablePathKey = "KAFKA_EMULATOR_EXECUTABLE_PATH"
- defaultPropertyPath = "."
- cacheRequestTimeoutKey = "CACHE_REQUEST_TIMEOUT"
- defaultCacheRequestTimeout = time.Second * 5
+ serverIpKey = "SERVER_IP"
+ serverPortKey = "SERVER_PORT"
+ beamSdkKey = "BEAM_SDK"
+ beamVersionKey = "BEAM_VERSION"
+ workingDirKey = "APP_WORK_DIR"
+ preparedModDirKey = "PREPARED_MOD_DIR"
+ numOfParallelJobsKey = "NUM_PARALLEL_JOBS"
+ cacheTypeKey = "CACHE_TYPE"
+ cacheAddressKey = "CACHE_ADDRESS"
+ beamPathKey = "BEAM_PATH"
+ cacheKeyExpirationTimeKey = "KEY_EXPIRATION_TIME"
+ pipelineExecuteTimeoutKey = "PIPELINE_EXPIRATION_TIMEOUT"
+ protocolTypeKey = "PROTOCOL_TYPE"
+ launchSiteKey = "LAUNCH_SITE"
+ projectIdKey = "GOOGLE_CLOUD_PROJECT"
+ pipelinesFolderKey = "PIPELINES_FOLDER_NAME"
+ defaultPipelinesFolder = "executable_files"
+ defaultLaunchSite = "local"
+ defaultProtocol = "HTTP"
+ defaultIp = "localhost"
+ defaultPort = 8080
+ defaultSdk = pb.Sdk_SDK_UNSPECIFIED
+ defaultBeamVersion = "<unknown>"
+ defaultBeamJarsPath = "/opt/apache/beam/jars/*"
+ defaultDatasetsPath = "/opt/playground/backend/datasets"
+ defaultKafkaEmulatorExecutablePath = "/opt/playground/backend/kafka-emulator/beam-playground-kafka-emulator.jar"
+ defaultCacheType = "local"
+ defaultCacheAddress = "localhost:6379"
+ defaultCacheKeyExpirationTime = time.Minute * 15
+ defaultPipelineExecuteTimeout = time.Minute * 10
+ jsonExt = ".json"
+ configFolderName = "configs"
+ defaultNumOfParallelJobs = 20
+ SDKConfigPathKey = "SDK_CONFIG"
+ defaultSDKConfigPath = "../sdks.yaml"
+ propertyPathKey = "PROPERTY_PATH"
+ datasetsPathKey = "DATASETS_PATH"
+ kafkaEmulatorExecutablePathKey = "KAFKA_EMULATOR_EXECUTABLE_PATH"
+ defaultPropertyPath = "."
+ cacheRequestTimeoutKey = "CACHE_REQUEST_TIMEOUT"
+ defaultCacheRequestTimeout = time.Second * 5
+ cleanupSnippetsFunctionsUrlKey = "CLEANUP_SNIPPETS_FUNCTIONS_URL"
+ defaultCleanupSnippetsFunctionsUrl = "http://cleanup_snippets:8080/"
+ putSnippetFunctionsUrlKey = "PUT_SNIPPET_FUNCTIONS_URL"
+ defaultPutSnippetFunctionsUrl = "http://put_snippet:8080/"
+ incrementSnippetViewsFunctionsUrlKey = "INCREMENT_SNIPPET_VIEWS_FUNCTIONS_URL"
+ defaultIncrementSnippetViewsFunctionsUrl = "http://increment_snippet_views:8080/"
)
// Environment operates with environment structures: NetworkEnvs, BeamEnvs, ApplicationEnvs
@@ -120,9 +126,31 @@ func GetApplicationEnvsFromOsEnvs() (*ApplicationEnvs, error) {
datasetsPath := getEnv(datasetsPathKey, defaultDatasetsPath)
kafkaEmulatorExecutablePath := getEnv(kafkaEmulatorExecutablePathKey, defaultKafkaEmulatorExecutablePath)
cacheRequestTimeout := getEnvAsDuration(cacheRequestTimeoutKey, defaultCacheRequestTimeout, "couldn't convert provided cache request timeout. Using default %s\n")
+ cleanupSnippetsFunctionsUrl := getEnv(cleanupSnippetsFunctionsUrlKey, defaultCleanupSnippetsFunctionsUrl)
+ putSnippetFunctionsUrl := getEnv(putSnippetFunctionsUrlKey, defaultPutSnippetFunctionsUrl)
+ incrementSnippetViewsFunctionsUrl := getEnv(incrementSnippetViewsFunctionsUrlKey, defaultIncrementSnippetViewsFunctionsUrl)
if value, present := os.LookupEnv(workingDirKey); present {
- return NewApplicationEnvs(value, launchSite, projectId, pipelinesFolder, sdkConfigPath, propertyPath, kafkaEmulatorExecutablePath, datasetsPath, NewCacheEnvs(cacheType, cacheAddress, cacheExpirationTime), pipelineExecuteTimeout, cacheRequestTimeout), nil
+ return NewApplicationEnvs(
+ value,
+ launchSite,
+ projectId,
+ pipelinesFolder,
+ sdkConfigPath,
+ propertyPath,
+ kafkaEmulatorExecutablePath,
+ datasetsPath,
+ cleanupSnippetsFunctionsUrl,
+ putSnippetFunctionsUrl,
+ incrementSnippetViewsFunctionsUrl,
+ NewCacheEnvs(
+ cacheType,
+ cacheAddress,
+ cacheExpirationTime,
+ ),
+ pipelineExecuteTimeout,
+ cacheRequestTimeout,
+ ), nil
}
return nil, errors.New("APP_WORK_DIR env should be provided with os.env")
}
diff --git a/playground/backend/internal/environment/environment_service_test.go b/playground/backend/internal/environment/environment_service_test.go
index ae8cf4a06e4..f74578aed57 100644
--- a/playground/backend/internal/environment/environment_service_test.go
+++ b/playground/backend/internal/environment/environment_service_test.go
@@ -103,9 +103,28 @@ func TestNewEnvironment(t *testing.T) {
want *Environment
}{
{name: "Create env service with default envs", want: &Environment{
- NetworkEnvs: *NewNetworkEnvs(defaultIp, defaultPort, defaultProtocol),
- BeamSdkEnvs: *NewBeamEnvs(defaultSdk, defaultBeamVersion, executorConfig, preparedModDir, 0),
- ApplicationEnvs: *NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout),
+ NetworkEnvs: *NewNetworkEnvs(defaultIp, defaultPort, defaultProtocol),
+ BeamSdkEnvs: *NewBeamEnvs(defaultSdk, defaultBeamVersion, executorConfig, preparedModDir, 0),
+ ApplicationEnvs: *NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ defaultCacheKeyExpirationTime,
+ },
+ defaultPipelineExecuteTimeout,
+ defaultCacheRequestTimeout,
+ ),
}},
}
for _, tt := range tests {
@@ -113,7 +132,26 @@ func TestNewEnvironment(t *testing.T) {
if got := NewEnvironment(
*NewNetworkEnvs(defaultIp, defaultPort, defaultProtocol),
*NewBeamEnvs(defaultSdk, defaultBeamVersion, executorConfig, preparedModDir, 0),
- *NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout)); !reflect.DeepEqual(got, tt.want) {
+ *NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ defaultCacheKeyExpirationTime,
+ },
+ defaultPipelineExecuteTimeout,
+ defaultCacheRequestTimeout,
+ )); !reflect.DeepEqual(got, tt.want) {
t.Errorf("NewEnvironment() = %v, want %v", got, tt.want)
}
})
@@ -223,8 +261,27 @@ func Test_getApplicationEnvsFromOsEnvs(t *testing.T) {
envsToSet map[string]string
}{
{
- name: "Working dir is provided",
- want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout),
+ name: "Working dir is provided",
+ want: NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ defaultCacheKeyExpirationTime,
+ },
+ defaultPipelineExecuteTimeout,
+ defaultCacheRequestTimeout,
+ ),
wantErr: false,
envsToSet: map[string]string{workingDirKey: "/app", launchSiteKey: defaultLaunchSite, projectIdKey: defaultProjectId},
},
@@ -234,26 +291,101 @@ func Test_getApplicationEnvsFromOsEnvs(t *testing.T) {
wantErr: true,
},
{
- name: "CacheKeyExpirationTimeKey is set with the correct value",
- want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, convertedTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout),
+ name: "CacheKeyExpirationTimeKey is set with the correct value",
+ want: NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ convertedTime,
+ },
+ defaultPipelineExecuteTimeout,
+ defaultCacheRequestTimeout),
wantErr: false,
envsToSet: map[string]string{workingDirKey: "/app", cacheKeyExpirationTimeKey: hour},
},
{
- name: "CacheKeyExpirationTimeKey is set with the incorrect value",
- want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout),
+ name: "CacheKeyExpirationTimeKey is set with the incorrect value",
+ want: NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ defaultCacheKeyExpirationTime,
+ },
+ defaultPipelineExecuteTimeout,
+ defaultCacheRequestTimeout,
+ ),
wantErr: false,
envsToSet: map[string]string{workingDirKey: "/app", cacheKeyExpirationTimeKey: "1"},
},
{
- name: "CacheKeyExpirationTimeKey is set with the correct value",
- want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, convertedTime, defaultCacheRequestTimeout),
+ name: "CacheKeyExpirationTimeKey is set with the correct value",
+ want: NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ defaultCacheKeyExpirationTime,
+ },
+ convertedTime,
+ defaultCacheRequestTimeout,
+ ),
wantErr: false,
envsToSet: map[string]string{workingDirKey: "/app", pipelineExecuteTimeoutKey: hour},
},
{
- name: "PipelineExecuteTimeoutKey is set with the incorrect value",
- want: NewApplicationEnvs("/app", defaultLaunchSite, defaultProjectId, defaultPipelinesFolder, defaultSDKConfigPath, defaultPropertyPath, defaultKafkaEmulatorExecutablePath, defaultDatasetsPath, &CacheEnvs{defaultCacheType, defaultCacheAddress, defaultCacheKeyExpirationTime}, defaultPipelineExecuteTimeout, defaultCacheRequestTimeout),
+ name: "PipelineExecuteTimeoutKey is set with the incorrect value",
+ want: NewApplicationEnvs(
+ "/app",
+ defaultLaunchSite,
+ defaultProjectId,
+ defaultPipelinesFolder,
+ defaultSDKConfigPath,
+ defaultPropertyPath,
+ defaultKafkaEmulatorExecutablePath,
+ defaultDatasetsPath,
+ defaultCleanupSnippetsFunctionsUrl,
+ defaultPutSnippetFunctionsUrl,
+ defaultIncrementSnippetViewsFunctionsUrl,
+ &CacheEnvs{
+ defaultCacheType,
+ defaultCacheAddress,
+ defaultCacheKeyExpirationTime,
+ },
+ defaultPipelineExecuteTimeout,
+ defaultCacheRequestTimeout,
+ ),
wantErr: false,
envsToSet: map[string]string{workingDirKey: "/app", pipelineExecuteTimeoutKey: "1"},
},
diff --git a/playground/backend/internal/environment/property.go b/playground/backend/internal/environment/property.go
index 8e9d034c097..db5a2917f9b 100644
--- a/playground/backend/internal/environment/property.go
+++ b/playground/backend/internal/environment/property.go
@@ -34,8 +34,6 @@ type Properties struct {
IdLength int8 `mapstructure:"id_length"`
// RemovingUnusedSnptsCron is the cron expression for the scheduled task to remove unused snippets
RemovingUnusedSnptsCron string `mapstructure:"removing_unused_snippets_cron"`
- // RemovingUnusedSnptsDays is the number of days after which a snippet becomes unused
- RemovingUnusedSnptsDays int32 `mapstructure:"removing_unused_snippets_days"`
}
func NewProperties(configPath string) (*Properties, error) {
diff --git a/playground/backend/internal/environment/property_test.go b/playground/backend/internal/environment/property_test.go
index 26d6d7c26e2..794eb1ec405 100644
--- a/playground/backend/internal/environment/property_test.go
+++ b/playground/backend/internal/environment/property_test.go
@@ -47,8 +47,7 @@ func TestNew(t *testing.T) {
if props.Salt != "Beam playground salt" ||
props.MaxSnippetSize != 1000000 ||
props.IdLength != 11 ||
- props.RemovingUnusedSnptsCron != "0 0 0 1 */1 *" ||
- props.RemovingUnusedSnptsDays != 180 {
+ props.RemovingUnusedSnptsCron != "0 0 0 1 */1 *" {
t.Errorf("NewProperties(): unexpected result")
}
})
diff --git a/playground/backend/internal/external_functions/external_functions_component.go b/playground/backend/internal/external_functions/external_functions_component.go
new file mode 100644
index 00000000000..5e6e416e67c
--- /dev/null
+++ b/playground/backend/internal/external_functions/external_functions_component.go
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package external_functions
+
+import (
+ "beam.apache.org/playground/backend/internal/utils"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+
+ "beam.apache.org/playground/backend/internal/db/entity"
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+)
+
+type ExternalFunctions interface {
+ // CleanupSnippets removes old snippets from the database.
+ CleanupSnippets(ctx context.Context) error
+
+ // PutSnippet puts the snippet to the database.
+ PutSnippet(ctx context.Context, snipId string, snippet *entity.Snippet) error
+
+ // IncrementSnippetViews increments the number of views for the snippet.
+ IncrementSnippetViews(ctx context.Context, snipId string) error
+}
+
+type externalFunctionsComponent struct {
+ cleanupSnippetsFunctionsUrl string
+ putSnippetFunctionsUrl string
+ incrementSnippetViewsFunctionsUrl string
+}
+
+func NewExternalFunctionsComponent(appEnvs environment.ApplicationEnvs) ExternalFunctions {
+ return &externalFunctionsComponent{
+ cleanupSnippetsFunctionsUrl: appEnvs.CleanupSnippetsFunctionsUrl(),
+ putSnippetFunctionsUrl: appEnvs.PutSnippetFunctionsUrl(),
+ incrementSnippetViewsFunctionsUrl: appEnvs.IncrementSnippetViewsFunctionsUrl(),
+ }
+}
+
+func makePostRequest(ctx context.Context, requestUrl string, body any) error {
+ var bodyReader io.Reader = nil
+
+ if body != nil {
+ bodyJson, err := json.Marshal(body)
+ if err != nil {
+ logger.Errorf("makePostRequest(): Couldn't marshal the body, err: %s\n", err.Error())
+ return err
+ }
+
+ bodyReader = bytes.NewReader(bodyJson)
+ } else {
+ bodyReader = bytes.NewReader([]byte("{}"))
+ }
+
+ request, err := http.NewRequestWithContext(ctx, "POST", requestUrl, bodyReader)
+ if err != nil {
+ logger.Errorf("makePostRequest(): Couldn't create the request, err: %s\n", err.Error())
+ return err
+ }
+
+ request.Header.Set("Content-Type", "application/json")
+
+ resp, err := http.DefaultClient.Do(request)
+ if err != nil {
+ logger.Errorf("makePostRequest(): Couldn't make POST request to the %s, err: %s\n", requestUrl, err.Error())
+ return err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("status code: %d", resp.StatusCode)
+ }
+
+ return nil
+}
+
+func (c *externalFunctionsComponent) CleanupSnippets(ctx context.Context) error {
+ namespace := utils.GetNamespace(ctx)
+ requestUrl := fmt.Sprintf("%s?namespace=%s", c.cleanupSnippetsFunctionsUrl, namespace)
+
+ if err := makePostRequest(ctx, requestUrl, nil); err != nil {
+ logger.Errorf("CleanupSnippets(): Couldn't cleanup snippets, err: %s\n", err.Error())
+ return err
+ }
+
+ return nil
+}
+
+func (c *externalFunctionsComponent) PutSnippet(ctx context.Context, snipId string, snippet *entity.Snippet) error {
+ namespace := utils.GetNamespace(ctx)
+ requestUrl := fmt.Sprintf("%s?namespace=%s&snipId=%s", c.putSnippetFunctionsUrl, namespace, snipId)
+
+ if err := makePostRequest(ctx, requestUrl, snippet); err != nil {
+ logger.Errorf("DeleteObsoleteSnippets(): Couldn't delete obsolete snippets, err: %s\n", err.Error())
+ return err
+ }
+
+ return nil
+}
+
+func (c *externalFunctionsComponent) IncrementSnippetViews(ctx context.Context, snipId string) error {
+ namespace := utils.GetNamespace(ctx)
+ requestUrl := fmt.Sprintf("%s?namespace=%s&snipId=%s", c.incrementSnippetViewsFunctionsUrl, namespace, snipId)
+
+ if err := makePostRequest(ctx, requestUrl, nil); err != nil {
+ logger.Errorf("IncrementSnippetViews(): Couldn't increment snippet views, err: %s\n", err.Error())
+ return err
+ }
+
+ return nil
+}
diff --git a/playground/backend/internal/tasks/task.go b/playground/backend/internal/tasks/task.go
index c77b4a06133..3d214bbdab3 100644
--- a/playground/backend/internal/tasks/task.go
+++ b/playground/backend/internal/tasks/task.go
@@ -16,12 +16,12 @@
package tasks
import (
+ "beam.apache.org/playground/backend/internal/external_functions"
"context"
"time"
"github.com/procyon-projects/chrono"
- "beam.apache.org/playground/backend/internal/db"
"beam.apache.org/playground/backend/internal/logger"
)
@@ -34,11 +34,11 @@ func New(ctx context.Context) *ScheduledTask {
return &ScheduledTask{ctx: ctx, taskScheduler: chrono.NewDefaultTaskScheduler()}
}
-func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, dayDiff int32, db db.Database) error {
+func (st *ScheduledTask) StartRemovingExtraSnippets(cron string, externalFunction external_functions.ExternalFunctions) error {
task, err := st.taskScheduler.ScheduleWithCron(func(ctx context.Context) {
logger.Info("ScheduledTask: StartRemovingExtraSnippets() is running...\n")
startDate := time.Now()
- if err := db.DeleteUnusedSnippets(ctx, dayDiff); err != nil {
+ if err := externalFunction.CleanupSnippets(ctx); err != nil {
logger.Errorf("ScheduledTask: StartRemovingExtraSnippets() error during deleting unused snippets, err: %s\n", err.Error())
}
diffTime := time.Now().Sub(startDate).Milliseconds()
diff --git a/playground/backend/internal/utils/datastore_utils.go b/playground/backend/internal/utils/datastore_utils.go
index 757350eb8fd..a4706189506 100644
--- a/playground/backend/internal/utils/datastore_utils.go
+++ b/playground/backend/internal/utils/datastore_utils.go
@@ -16,6 +16,7 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/logger"
"context"
"os"
"strconv"
@@ -93,7 +94,11 @@ func getNameKey(ctx context.Context, kind, id string, parentId *datastore.Key) *
}
func GetNamespace(ctx context.Context) string {
- namespace, ok := ctx.Value(constants.DatastoreNamespaceKey).(string)
+ namespaceValue := ctx.Value(constants.DatastoreNamespaceKey)
+ namespace, ok := namespaceValue.(string)
+ if namespaceValue != nil && !ok {
+ logger.Warnf("GetNamespace(): %s value is set in context, but is not a string", constants.DatastoreNamespaceKey)
+ }
if !ok {
namespace, ok = os.LookupEnv(constants.DatastoreNamespaceKey)
if !ok {
diff --git a/playground/backend/playground_functions/Dockerfile b/playground/backend/playground_functions/Dockerfile
new file mode 100644
index 00000000000..a4045c02131
--- /dev/null
+++ b/playground/backend/playground_functions/Dockerfile
@@ -0,0 +1,38 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+# This Dockerfile is only for local testing
+
+FROM golang:1.20-alpine as build
+
+COPY . /app
+WORKDIR /app/playground_functions
+
+RUN ls -la
+
+RUN go mod download
+
+RUN go build -o /app/cloudfunction ./cmd
+
+FROM alpine:3.17
+
+COPY --from=build /app/cloudfunction /app/cloudfunction
+
+EXPOSE 8080
+
+ENTRYPOINT ["/app/cloudfunction"]
diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/playground_functions/cmd/main.go
similarity index 66%
copy from playground/backend/internal/db/entity/schema.go
copy to playground/backend/playground_functions/cmd/main.go
index 43aaacfca34..f4946694bfc 100644
--- a/playground/backend/internal/db/entity/schema.go
+++ b/playground/backend/playground_functions/cmd/main.go
@@ -13,8 +13,23 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package entity
+package main
-type SchemaEntity struct {
- Descr string `datastore:"descr,noindex"`
+import (
+ "log"
+ "os"
+
+ _ "beam.apache.org/playground/backend"
+ "github.com/GoogleCloudPlatform/functions-framework-go/funcframework"
+)
+
+func main() {
+ // Use PORT environment variable, or default to 8080.
+ port := "8080"
+ if envPort := os.Getenv("PORT"); envPort != "" {
+ port = envPort
+ }
+ if err := funcframework.Start(port); err != nil {
+ log.Fatalf("funcframework.Start: %v\n", err)
+ }
}
diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/playground_functions/func_enviornment.go
similarity index 69%
copy from playground/backend/internal/db/entity/schema.go
copy to playground/backend/playground_functions/func_enviornment.go
index 43aaacfca34..f86b55bfe84 100644
--- a/playground/backend/internal/db/entity/schema.go
+++ b/playground/backend/playground_functions/func_enviornment.go
@@ -13,8 +13,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package entity
+package playground_functions
-type SchemaEntity struct {
- Descr string `datastore:"descr,noindex"`
+import "os"
+
+type Environment interface {
+ GetProjectId() string
+}
+
+type environment struct {
+ projectID string
+}
+
+func GetEnvironment() Environment {
+ projectId := os.Getenv("GOOGLE_CLOUD_PROJECT")
+ return &environment{
+ projectID: projectId,
+ }
+}
+
+func (e *environment) GetProjectId() string {
+ return e.projectID
}
diff --git a/playground/backend/internal/db/entity/schema.go b/playground/backend/playground_functions/middleware.go
similarity index 63%
copy from playground/backend/internal/db/entity/schema.go
copy to playground/backend/playground_functions/middleware.go
index 43aaacfca34..8927ea1ec82 100644
--- a/playground/backend/internal/db/entity/schema.go
+++ b/playground/backend/playground_functions/middleware.go
@@ -13,8 +13,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package entity
+package playground_functions
-type SchemaEntity struct {
- Descr string `datastore:"descr,noindex"`
+import "net/http"
+
+// EnsureMethod is a middleware method which will only allow requests with the specified method to pass through.
+func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc {
+ return func(next http.HandlerFunc) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == method {
+ next(w, r)
+ } else {
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ }
+ }
+ }
}
diff --git a/playground/backend/properties.yaml b/playground/backend/properties.yaml
index 12f5adb2218..67d969ce370 100644
--- a/playground/backend/properties.yaml
+++ b/playground/backend/properties.yaml
@@ -23,5 +23,3 @@ id_length: 11
# Check your cron expression here: https://crontab.cronhub.io/
# Time trigger for the scheduled task will be updated every time the application is restarted
removing_unused_snippets_cron: 0 0 0 1 */1 *
-# Number of days after which a snippet becomes unused
-removing_unused_snippets_days: 180
diff --git a/playground/docker-compose.local.yaml b/playground/docker-compose.local.yaml
index 92121a2f0c5..a6c38925e31 100644
--- a/playground/docker-compose.local.yaml
+++ b/playground/docker-compose.local.yaml
@@ -29,6 +29,45 @@ services:
ports:
- "8081:8081"
+ cleanup_snippets:
+ build:
+ context: ./backend
+ dockerfile: ./playground_functions/Dockerfile
+ environment:
+ GOOGLE_CLOUD_PROJECT: test
+ DATASTORE_EMULATOR_HOST: datastore:8081
+ FUNCTION_TARGET: cleanupSnippets
+ expose:
+ - "8080"
+ depends_on:
+ - datastore
+
+ put_snippet:
+ build:
+ context: ./backend
+ dockerfile: ./playground_functions/Dockerfile
+ environment:
+ GOOGLE_CLOUD_PROJECT: test
+ DATASTORE_EMULATOR_HOST: datastore:8081
+ FUNCTION_TARGET: putSnippet
+ expose:
+ - "8080"
+ depends_on:
+ - datastore
+
+ increment_snippet_views:
+ build:
+ context: ./backend
+ dockerfile: ./playground_functions/Dockerfile
+ environment:
+ GOOGLE_CLOUD_PROJECT: test
+ DATASTORE_EMULATOR_HOST: datastore:8081
+ FUNCTION_TARGET: incrementSnippetViews
+ expose:
+ - "8080"
+ depends_on:
+ - datastore
+
router:
image: apache/beam_playground-backend-router
environment:
@@ -38,11 +77,15 @@ services:
CACHE_ADDRESS: redis:6379
SDK_CONFIG: /opt/playground/backend/sdks-emulator.yaml
SERVER_PORT: 8082
+ APPLY_MIGRATIONS: "True"
ports:
- "8082:8082"
depends_on:
- redis
- datastore
+ - increment_snippet_views
+ - put_snippet
+ - cleanup_snippets
go_runner:
image: apache/beam_playground-backend-go
diff --git a/playground/index.yaml b/playground/index.yaml
index 184b8ad3a2d..85461605c18 100644
--- a/playground/index.yaml
+++ b/playground/index.yaml
@@ -23,4 +23,3 @@ indexes:
properties:
- name: persistenceKey
- name: numberOfFiles
-
diff --git a/playground/infrastructure/helm-playground/templates/deployment-router.yml b/playground/infrastructure/helm-playground/templates/deployment-router.yml
index 7b2d20dca2c..e505a849589 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-router.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-router.yml
@@ -44,6 +44,12 @@ spec:
value: "5"
- name: DATASTORE_NAMESPACE
value: {{ .Values.datastore_name }}
+ - name: CLEANUP_SNIPPETS_FUNCTIONS_URL
+ value: {{ .Values.func_clean }}
+ - name: PUT_SNIPPET_FUNCTIONS_URL
+ value: {{ .Values.func_put }}
+ - name: INCREMENT_SNIPPET_VIEWS_FUNCTIONS_URL
+ value: {{ .Values.func_view }}
livenessProbe:
httpGet:
path: /liveness
diff --git a/playground/terraform/README.md b/playground/terraform/README.md
index 4a3db70600c..3ff1b62bef3 100644
--- a/playground/terraform/README.md
+++ b/playground/terraform/README.md
@@ -31,7 +31,9 @@ Ensure that the account has at least following privileges:
- App Engine Creator
- Artifact Registry Administrator
- Cloud Datastore Index Admin
+ - Cloud Datastore User
- Cloud Memorystore Redis Admin
+ - Cloud Functions Developer
- Compute Admin
- Create Service Accounts
- DNS Administrator
@@ -55,6 +57,7 @@ Ensure that the account has at least following privileges:
* [Terraform](https://www.terraform.io/downloads)
* [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk)
* [Kubectl authentication](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke)
+* [GO](https://go.dev/doc/install)
6. Apache Beam Git repository cloned locally
@@ -76,10 +79,11 @@ redis_tier = "BASIC" # Redis tier type. Options:
min_count = 2 # Min node count for the GKE cluster
max_count = 6 # Max node count for the GKE cluster
skip_appengine_deploy = false # AppEngine flag - defined if AppEngine and Datastore need to be installed. Should be "true" if AppEngine and Datastore were installed before
-ip-address-name = "playground-static-ip" # GCP Static IP Address name
+ip_address_name = "playground-static-ip" # GCP Static IP Address name
repository_id = "playground-artifacts" # GCP Artifact repository name for Playground images
-service_account_id = "playground-gke-account" # GCP Service account name
+service_account_id = "playground-gke-sa" # GCP Service account name
gke_machine_type = "e2-standard-8" # Machine type for GKE Nodes
+env = "prod" # Environment. The same value as for <environment_name> parameter
```
* `state.tfbackend` environment variables:
diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts
index 8e1e5b48fca..4fdb1007b35 100644
--- a/playground/terraform/build.gradle.kts
+++ b/playground/terraform/build.gradle.kts
@@ -267,6 +267,51 @@ tasks.register<TerraformTask>("setPlaygroundStaticIpAddressName") {
}
}
+tasks.register<TerraformTask>("setPlaygroundFunctionCleanupUrl") {
+ group = "deploy"
+
+ dependsOn("terraformInit")
+ dependsOn("terraformRef")
+
+ args("output", "playground_function_cleanup_url")
+ standardOutput = ByteArrayOutputStream()
+
+ doLast {
+ project.rootProject.extra["playground_function_cleanup_url"] =
+ standardOutput.toString().trim().replace("\"", "")
+ }
+}
+
+tasks.register<TerraformTask>("setPlaygroundFunctionPutUrl") {
+ group = "deploy"
+
+ dependsOn("terraformInit")
+ dependsOn("terraformRef")
+
+ args("output", "playground_function_put_url")
+ standardOutput = ByteArrayOutputStream()
+
+ doLast {
+ project.rootProject.extra["playground_function_put_url"] =
+ standardOutput.toString().trim().replace("\"", "")
+ }
+}
+
+tasks.register<TerraformTask>("setPlaygroundFunctionViewUrl") {
+ group = "deploy"
+
+ dependsOn("terraformInit")
+ dependsOn("terraformRef")
+
+ args("output", "playground_function_view_url")
+ standardOutput = ByteArrayOutputStream()
+
+ doLast {
+ project.rootProject.extra["playground_function_view_url"] =
+ standardOutput.toString().trim().replace("\"", "")
+ }
+}
+
tasks.register("takeConfig") {
group = "deploy"
@@ -274,6 +319,9 @@ tasks.register("takeConfig") {
dependsOn("setPlaygroundRedisIp")
dependsOn("setPlaygroundGkeProject")
dependsOn("setPlaygroundStaticIpAddressName")
+ dependsOn("setPlaygroundFunctionCleanupUrl")
+ dependsOn("setPlaygroundFunctionPutUrl")
+ dependsOn("setPlaygroundFunctionViewUrl")
doLast {
var d_tag = ""
@@ -314,6 +362,9 @@ tasks.register("takeConfig") {
val registry = project.rootProject.extra["docker-repository-root"]
val ipaddrname = project.rootProject.extra["playground_static_ip_address_name"]
val datastore_name = if (project.hasProperty("datastore-namespace")) (project.property("datastore-namespace") as String) else ""
+ val pgfuncclean = project.rootProject.extra["playground_function_cleanup_url"]
+ val pgfuncput = project.rootProject.extra["playground_function_put_url"]
+ val pgfuncview = project.rootProject.extra["playground_function_view_url"]
file.appendText(
"""
@@ -325,11 +376,32 @@ static_ip_name: ${ipaddrname}
tag: $d_tag
datastore_name: ${datastore_name}
dns_name: ${dns_name}
+func_clean: ${pgfuncclean}
+func_put: ${pgfuncput}
+func_view: ${pgfuncview}
"""
)
}
}
+
+task("applyMigrations") {
+ doLast {
+ val namespace = if (project.hasProperty("datastore-namespace")) (project.property("datastore-namespace") as String) else ""
+ val projectId = project.rootProject.extra["playground_gke_project"]
+ val modulePath = project(":playground").projectDir.absolutePath
+ val sdkConfig = "$modulePath/sdks.yaml"
+ exec {
+ workingDir("$modulePath/backend")
+ executable("go")
+ args("run", "cmd/migration_tool/migration_tool.go",
+ "-project-id", projectId,
+ "-sdk-config", sdkConfig,
+ "-namespace", namespace)
+ }
+ }
+}
+
tasks.register("helmRelease") {
group = "deploy"
val modulePath = project(":playground").projectDir.absolutePath
@@ -339,7 +411,7 @@ tasks.register("helmRelease") {
executable("helm")
args("upgrade", "--install", "playground", "$helmdir")
}
- }
+ }
}
tasks.register("gkebackend") {
@@ -350,15 +422,18 @@ tasks.register("gkebackend") {
val pushFrontTask = tasks.getByName("pushFront")
val indexcreateTask = tasks.getByName("indexcreate")
val helmTask = tasks.getByName("helmRelease")
+ var applyMigrations = tasks.getByName("applyMigrations")
dependsOn(initTask)
dependsOn(takeConfigTask)
dependsOn(pushBackTask)
dependsOn(pushFrontTask)
dependsOn(indexcreateTask)
+ dependsOn(applyMigrations)
dependsOn(helmTask)
takeConfigTask.mustRunAfter(initTask)
pushBackTask.mustRunAfter(takeConfigTask)
pushFrontTask.mustRunAfter(pushBackTask)
indexcreateTask.mustRunAfter(pushFrontTask)
- helmTask.mustRunAfter(indexcreateTask)
+ applyMigrations.mustRunAfter(indexcreateTask)
+ helmTask.mustRunAfter(applyMigrations)
}
diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/api_enable/variables.tf
index f0c0e5823f3..41eef1e165f 100644
--- a/playground/terraform/infrastructure/api_enable/variables.tf
+++ b/playground/terraform/infrastructure/api_enable/variables.tf
@@ -23,5 +23,5 @@ variable "project_id" {
variable "services" {
description = "Enable necessary APIs in GCP"
- default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"]
+ default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","cloudfunctions.googleapis.com","cloudbuild.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"]
}
diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/archive_file/main.tf
similarity index 80%
copy from playground/terraform/infrastructure/ip_address/main.tf
copy to playground/terraform/infrastructure/archive_file/main.tf
index 3e2a8f3524a..90ce40ea149 100644
--- a/playground/terraform/infrastructure/ip_address/main.tf
+++ b/playground/terraform/infrastructure/archive_file/main.tf
@@ -17,6 +17,12 @@
# under the License.
#
-resource "google_compute_global_address" "pg-ip" {
- name = var.ip-address-name
+data "archive_file" "backend_folder" {
+ type = "zip"
+ source_dir = "${path.root}/../backend/"
+ output_path = "${path.root}/../cloudfunction.zip"
+
+ excludes = [
+ "containers"
+ ]
}
diff --git a/playground/terraform/infrastructure/cloudfunctions/main.tf b/playground/terraform/infrastructure/cloudfunctions/main.tf
new file mode 100644
index 00000000000..690b032f3b8
--- /dev/null
+++ b/playground/terraform/infrastructure/cloudfunctions/main.tf
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+locals {
+ functions = [
+ {
+ name = "playground-function-cleanup-${var.env}"
+ description = "Playground function cleanup-${var.env}"
+ entry_point = "cleanupSnippets"
+ },
+ {
+ name = "playground-function-delete-${var.env}"
+ description = "Playground function delete-${var.env}"
+ entry_point = "putSnippet"
+ },
+ {
+ name = "playground-function-view-${var.env}"
+ description = "Playground function view-${var.env}"
+ entry_point = "incrementSnippetViews"
+ },
+ ]
+}
+
+resource "google_cloudfunctions_function" "playground_functions" {
+ count = length(local.functions)
+ name = local.functions[count.index].name
+ description = local.functions[count.index].description
+ entry_point = local.functions[count.index].entry_point
+ ingress_settings = "ALLOW_INTERNAL_ONLY"
+ runtime = "go120"
+ source_archive_bucket = var.gkebucket
+ source_archive_object = "cloudfunction.zip"
+ trigger_http = true
+ timeout = "540"
+ available_memory_mb = 2048
+ service_account_email = var.service_account_email_cf
+ environment_variables = {
+ GOOGLE_CLOUD_PROJECT = var.project_id
+ }
+}
+
+resource "google_cloudfunctions_function_iam_member" "invokers" {
+ count = length(local.functions)
+ project = var.project_id
+ region = var.region
+ cloud_function = google_cloudfunctions_function.playground_functions[count.index].name
+ role = "roles/cloudfunctions.invoker"
+ member = "allUsers"
+}
\ No newline at end of file
diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/cloudfunctions/output.tf
similarity index 68%
copy from playground/terraform/infrastructure/api_enable/variables.tf
copy to playground/terraform/infrastructure/cloudfunctions/output.tf
index f0c0e5823f3..a55636624d0 100644
--- a/playground/terraform/infrastructure/api_enable/variables.tf
+++ b/playground/terraform/infrastructure/cloudfunctions/output.tf
@@ -17,11 +17,14 @@
# under the License.
#
-variable "project_id" {
- description = "project_id"
+output "playground_function_cleanup_url" {
+ value = google_cloudfunctions_function.playground_functions[0].https_trigger_url
}
-variable "services" {
- description = "Enable necessary APIs in GCP"
- default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"]
+output "playground_function_put_url" {
+ value = google_cloudfunctions_function.playground_functions[1].https_trigger_url
+}
+
+output "playground_function_view_url" {
+ value = google_cloudfunctions_function.playground_functions[2].https_trigger_url
}
diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/cloudfunctions/variables.tf
similarity index 61%
copy from playground/terraform/infrastructure/api_enable/variables.tf
copy to playground/terraform/infrastructure/cloudfunctions/variables.tf
index f0c0e5823f3..9d3f86f63a6 100644
--- a/playground/terraform/infrastructure/api_enable/variables.tf
+++ b/playground/terraform/infrastructure/cloudfunctions/variables.tf
@@ -17,11 +17,27 @@
# under the License.
#
+variable "env" {
+ description = "CloudFunction Environment"
+}
+
+variable "function_description" {
+ type = string
+ default = "Playground function"
+}
+
+variable "gkebucket" {
+ description = "Bucket name for CloudFunction"
+}
+
+variable "service_account_email_cf" {
+ description = "Service account email for CloudFunctions"
+}
+
variable "project_id" {
- description = "project_id"
+ description = "The GCP Project ID where Playground Applications will be created"
}
-variable "services" {
- description = "Enable necessary APIs in GCP"
- default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"]
+variable "region" {
+ description = "The GCP Project region where Cloudfunctions will be created"
}
diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/gke_bucket/main.tf
similarity index 66%
copy from playground/terraform/infrastructure/api_enable/variables.tf
copy to playground/terraform/infrastructure/gke_bucket/main.tf
index f0c0e5823f3..859a01d4443 100644
--- a/playground/terraform/infrastructure/api_enable/variables.tf
+++ b/playground/terraform/infrastructure/gke_bucket/main.tf
@@ -17,11 +17,19 @@
# under the License.
#
-variable "project_id" {
- description = "project_id"
+resource "google_storage_bucket" "bucket" {
+ name = "${var.bucket_name}-cloudbuild"
+ location = var.region
+ force_destroy = true
}
-variable "services" {
- description = "Enable necessary APIs in GCP"
- default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"]
+resource "google_storage_bucket_object" "cloudfunction_object" {
+ name = "cloudfunction.zip"
+ bucket = google_storage_bucket.bucket.name
+
+ source = "${path.root}/../cloudfunction.zip"
+
+ content_type = "application/zip"
+ content_encoding = "zip"
+
}
diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/gke_bucket/output.tf
similarity index 89%
copy from playground/terraform/infrastructure/ip_address/main.tf
copy to playground/terraform/infrastructure/gke_bucket/output.tf
index 3e2a8f3524a..45da0edd96b 100644
--- a/playground/terraform/infrastructure/ip_address/main.tf
+++ b/playground/terraform/infrastructure/gke_bucket/output.tf
@@ -17,6 +17,6 @@
# under the License.
#
-resource "google_compute_global_address" "pg-ip" {
- name = var.ip-address-name
+output "playground_google_storage_bucket" {
+ value = google_storage_bucket.bucket.name
}
diff --git a/playground/terraform/infrastructure/ip_address/variables.tf b/playground/terraform/infrastructure/gke_bucket/variables.tf
similarity index 83%
copy from playground/terraform/infrastructure/ip_address/variables.tf
copy to playground/terraform/infrastructure/gke_bucket/variables.tf
index 42f60ed7ae3..45de6ab147c 100644
--- a/playground/terraform/infrastructure/ip_address/variables.tf
+++ b/playground/terraform/infrastructure/gke_bucket/variables.tf
@@ -17,7 +17,10 @@
# under the License.
#
-variable "ip-address-name" {
- description = "Static IP address name"
- default = "pg-static-ip"
+variable "region" {
+ description = "Region of Playground Examples Bucket"
+}
+
+variable "bucket_name" {
+ description = "Bucket name for CloudFunction"
}
diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/ip_address/main.tf
index 3e2a8f3524a..c649d35a8fe 100644
--- a/playground/terraform/infrastructure/ip_address/main.tf
+++ b/playground/terraform/infrastructure/ip_address/main.tf
@@ -18,5 +18,5 @@
#
resource "google_compute_global_address" "pg-ip" {
- name = var.ip-address-name
+ name = var.ip_address_name
}
diff --git a/playground/terraform/infrastructure/ip_address/variables.tf b/playground/terraform/infrastructure/ip_address/variables.tf
index 42f60ed7ae3..dea28526da6 100644
--- a/playground/terraform/infrastructure/ip_address/variables.tf
+++ b/playground/terraform/infrastructure/ip_address/variables.tf
@@ -17,7 +17,7 @@
# under the License.
#
-variable "ip-address-name" {
+variable "ip_address_name" {
description = "Static IP address name"
default = "pg-static-ip"
}
diff --git a/playground/terraform/infrastructure/main.tf b/playground/terraform/infrastructure/main.tf
index edb32a36857..75dfab7799d 100644
--- a/playground/terraform/infrastructure/main.tf
+++ b/playground/terraform/infrastructure/main.tf
@@ -49,6 +49,29 @@ module "artifact_registry" {
location = var.repository_location
}
+module "gke_bucket" {
+ depends_on = [module.setup, module.network, module.api_enable, module.ip_address, module.archive_file]
+ source = "./gke_bucket"
+ region = var.region
+ bucket_name = var.state_bucket
+
+}
+
+module "archive_file" {
+ depends_on = [module.setup, module.network, module.api_enable, module.ip_address]
+ source = "./archive_file"
+}
+
+module "cloudfunctions" {
+ depends_on = [module.setup, module.network, module.api_enable, module.ip_address, module.gke_bucket]
+ source = "./cloudfunctions"
+ gkebucket = module.gke_bucket.playground_google_storage_bucket
+ project_id = var.project_id
+ service_account_email_cf = module.setup.service_account_email_cf
+ region = var.region
+ env = var.env
+}
+
module "memorystore" {
depends_on = [module.setup, module.network, module.api_enable, module.ip_address]
source = "./memorystore"
@@ -81,7 +104,7 @@ module "gke" {
module "ip_address" {
source = "./ip_address"
depends_on = [module.setup, module.api_enable]
- ip-address-name = var.ip-address-name
+ ip_address_name = var.ip_address_name
}
module "appengine" {
@@ -106,6 +129,7 @@ module "private_dns" {
private_zones = [
"gcr.io",
"pkg.dev",
- "cloud.google.com"
+ "cloud.google.com",
+ "cloudfunctions.net"
]
-}
\ No newline at end of file
+}
diff --git a/playground/terraform/infrastructure/output.tf b/playground/terraform/infrastructure/output.tf
index 38d61b7887f..b4e1e6161dc 100644
--- a/playground/terraform/infrastructure/output.tf
+++ b/playground/terraform/infrastructure/output.tf
@@ -63,4 +63,16 @@ output "playground_gke_project" {
output "playground_static_ip_address_name" {
value = module.ip_address.playground_static_ip_address_name
-}
\ No newline at end of file
+}
+
+output "playground_function_cleanup_url" {
+ value = module.cloudfunctions.playground_function_cleanup_url
+}
+
+output "playground_function_put_url" {
+ value = module.cloudfunctions.playground_function_put_url
+}
+
+output "playground_function_view_url" {
+ value = module.cloudfunctions.playground_function_view_url
+}
diff --git a/playground/terraform/infrastructure/setup/iam.tf b/playground/terraform/infrastructure/setup/iam.tf
index ec31f26bf1b..55bfdf215d6 100644
--- a/playground/terraform/infrastructure/setup/iam.tf
+++ b/playground/terraform/infrastructure/setup/iam.tf
@@ -52,11 +52,25 @@ resource "google_service_account" "playground_service_account" {
display_name = var.service_account_id
}
+resource "google_service_account" "playground_service_account_cf" {
+ account_id = "${google_service_account.playground_service_account.account_id}-cf"
+ display_name = "${google_service_account.playground_service_account.account_id}-cf"
+}
+
resource "google_project_iam_member" "terraform_service_account_roles" {
for_each = toset([
- "roles/container.admin", "roles/artifactregistry.reader", "roles/datastore.owner", "roles/redis.admin",
+ "roles/container.developer", "roles/artifactregistry.reader", "roles/datastore.viewer", "roles/redis.serviceAgent", "roles/redis.viewer",
])
- role = each.key
+ role = each.value
member = "serviceAccount:${google_service_account.playground_service_account.email}"
project = var.project_id
}
+
+resource "google_project_iam_member" "cloudfunction" {
+ for_each = toset([
+ "roles/storage.objectViewer","roles/cloudfunctions.invoker","roles/datastore.user",
+ ])
+ role = each.key
+ member = "serviceAccount:${google_service_account.playground_service_account_cf.email}"
+ project = var.project_id
+}
diff --git a/playground/terraform/infrastructure/setup/output.tf b/playground/terraform/infrastructure/setup/output.tf
index 547bc4fe977..c4f74e6d986 100644
--- a/playground/terraform/infrastructure/setup/output.tf
+++ b/playground/terraform/infrastructure/setup/output.tf
@@ -19,4 +19,8 @@
output "service_account_email" {
value = google_service_account.playground_service_account.email
-}
\ No newline at end of file
+}
+
+output "service_account_email_cf" {
+ value = google_service_account.playground_service_account_cf.email
+}
diff --git a/playground/terraform/infrastructure/variables.tf b/playground/terraform/infrastructure/variables.tf
index 80fddf807c9..f3f9efe75ca 100644
--- a/playground/terraform/infrastructure/variables.tf
+++ b/playground/terraform/infrastructure/variables.tf
@@ -31,6 +31,10 @@ variable "region" {
description = "Infrastructure Region"
}
+variable "env" {}
+
+variable "state_bucket" {}
+
#IAM
variable "service_account_id" {
@@ -129,7 +133,7 @@ variable "network_region" {
default = "us-central1"
}
-variable "ip-address-name" {
+variable "ip_address_name" {
description = "Static IP address name"
default = "pg-static-ip"
}
diff --git a/playground/terraform/main.tf b/playground/terraform/main.tf
index e852ba9ffc6..a91e875736a 100644
--- a/playground/terraform/main.tf
+++ b/playground/terraform/main.tf
@@ -20,12 +20,14 @@
module "infrastructure" {
source = "./infrastructure"
project_id = var.project_id
- environment = var.environment
region = var.region
+ environment = var.environment
network_region = var.region
redis_region = var.region
location = var.zone
service_account_id = var.service_account_id
+ state_bucket = var.state_bucket
+ env = var.env
#Artifact Registry
repository_id = var.repository_id
repository_location = var.region
@@ -37,7 +39,7 @@ module "infrastructure" {
redis_memory_size_gb = var.redis_memory_size_gb
#NETWORK
network_name = var.network_name
- ip-address-name = var.ip-address-name
+ ip_address_name = var.ip_address_name
subnetwork_name = var.subnetwork_name
#GKE
gke_machine_type = var.gke_machine_type
diff --git a/playground/terraform/output.tf b/playground/terraform/output.tf
index d60af591eca..38f1859219e 100644
--- a/playground/terraform/output.tf
+++ b/playground/terraform/output.tf
@@ -63,4 +63,16 @@ output "playground_gke_project" {
output "playground_static_ip_address_name" {
value = module.infrastructure.playground_static_ip_address_name
-}
\ No newline at end of file
+}
+
+output "playground_function_cleanup_url" {
+ value = module.infrastructure.playground_function_cleanup_url
+}
+
+output "playground_function_put_url" {
+ value = module.infrastructure.playground_function_put_url
+}
+
+output "playground_function_view_url" {
+ value = module.infrastructure.playground_function_view_url
+}
diff --git a/playground/terraform/variables.tf b/playground/terraform/variables.tf
index a8b3c9144ab..13d467aac51 100644
--- a/playground/terraform/variables.tf
+++ b/playground/terraform/variables.tf
@@ -27,6 +27,10 @@ variable "environment" {
description = "prod,dev,beta"
}
+variable "env" {
+ description = "prod,dev,beta"
+}
+
variable "region" {
description = "Infrastructure Region"
}
@@ -35,6 +39,7 @@ variable "zone" {
description = "Infrastructure Zone"
}
+variable "state_bucket" {}
# Infrastructure variables
#GKE
@@ -77,7 +82,7 @@ variable "service_account_id" {
}
#Network
-variable "ip-address-name" {
+variable "ip_address_name" {
description = "Static IP address name"
default = "pg-static-ip"
}