You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2022/01/12 09:29:29 UTC
[servicecomb-service-center] branch master updated: [feat] add syncer delete expire tombstone data (#1214)
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-service-center.git
The following commit(s) were added to refs/heads/master by this push:
new 05fe442 [feat] add syncer delete expire tombstone data (#1214)
05fe442 is described below
commit 05fe44290d56c1fb585559aded3188e25e56a61c
Author: xiaoluoluo <78...@qq.com>
AuthorDate: Wed Jan 12 17:29:24 2022 +0800
[feat] add syncer delete expire tombstone data (#1214)
* [feat] add syncer delete expire tombstone data
* [feat] add syncer delete expire tombstone data
Co-authored-by: 1162641716@qq.com <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>
---
etc/conf/syncer.yaml | 5 ++
server/core/microservice.go | 6 +-
syncer/bootstrap/bootstrap.go | 1 +
syncer/job/tombstone/tombstone.go | 67 +++++++++++++++++++
syncer/service/tombstone/tombstone.go | 72 ++++++++++++++++++++
syncer/service/tombstone/tombstone_test.go | 102 +++++++++++++++++++++++++++++
6 files changed, 250 insertions(+), 3 deletions(-)
diff --git a/etc/conf/syncer.yaml b/etc/conf/syncer.yaml
index d4fa7c1..784f86a 100644
--- a/etc/conf/syncer.yaml
+++ b/etc/conf/syncer.yaml
@@ -6,3 +6,8 @@ sync:
endpoints: ["127.0.0.1:30105"]
# only allow mode implemented in incremental approach like push, watch(such as pub/sub, long polling)
mode: [push]
+ tombstone:
+ retire:
+ # use linux crontab not Quartz cron
+ cron:
+ reserve: 24h
\ No newline at end of file
diff --git a/server/core/microservice.go b/server/core/microservice.go
index 3f0684f..e37433c 100644
--- a/server/core/microservice.go
+++ b/server/core/microservice.go
@@ -74,9 +74,9 @@ func InitRegistration() {
},
}
- name := config.GetString("registry.instance.datacenter.name", " ")
- region := config.GetString("registry.instance.datacenter.region", " ")
- availableZone := config.GetString("registry.instance.datacenter.availableZone", " ")
+ name := config.GetString("registry.instance.datacenter.name", "")
+ region := config.GetString("registry.instance.datacenter.region", "")
+ availableZone := config.GetString("registry.instance.datacenter.availableZone", "")
if len(name) > 0 && len(region) > 0 && len(availableZone) > 0 {
Instance.DataCenterInfo = &discovery.DataCenterInfo{
Name: name,
diff --git a/syncer/bootstrap/bootstrap.go b/syncer/bootstrap/bootstrap.go
index fc6c6e6..d55bc1f 100644
--- a/syncer/bootstrap/bootstrap.go
+++ b/syncer/bootstrap/bootstrap.go
@@ -18,6 +18,7 @@
package bootstrap
import (
+ _ "github.com/apache/servicecomb-service-center/syncer/job/tombstone"
_ "github.com/apache/servicecomb-service-center/syncer/resource"
)
diff --git a/syncer/job/tombstone/tombstone.go b/syncer/job/tombstone/tombstone.go
new file mode 100644
index 0000000..8c6f120
--- /dev/null
+++ b/syncer/job/tombstone/tombstone.go
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tombstone
+
+import (
+ "fmt"
+
+ "github.com/apache/servicecomb-service-center/pkg/log"
+ "github.com/apache/servicecomb-service-center/server/config"
+ "github.com/apache/servicecomb-service-center/server/service/dlock"
+ "github.com/apache/servicecomb-service-center/syncer/service/tombstone"
+ "github.com/robfig/cron/v3"
+)
+
+const (
+ defaultDeleteExpireTombstoneCron = "0 0 * * *" // once every day at 00:00
+ deleteExpireTombstoneLockTTL = 60
+ deleteExpireTombstoneLockKey = "delete-expireTombstone-job"
+)
+
+func init() {
+ cronStr := config.GetString("sync.tombstone.retire,cron", "")
+ if len(cronStr) <= 0 {
+ cronStr = defaultDeleteExpireTombstoneCron
+ }
+
+ log.Info(fmt.Sprintf("start syncer tombstone job, plan is %v", cronStr))
+ c := cron.New()
+ _, err := c.AddFunc(cronStr, func() {
+ deleteExpireTombStone()
+ })
+ if err != nil {
+ log.Error("cron add func failed", err)
+ return
+ }
+ c.Start()
+}
+
+func deleteExpireTombStone() {
+ err := dlock.TryLock(deleteExpireTombstoneLockKey, deleteExpireTombstoneLockTTL)
+ if err != nil {
+ log.Error(fmt.Sprintf("try lock %s failed", deleteExpireTombstoneLockKey), err)
+ return
+ }
+ defer dlock.Unlock(deleteExpireTombstoneLockKey)
+
+ log.Info("start delete expire tombstone job")
+ err = tombstone.DeleteExpireTombStone()
+ if err != nil {
+ log.Error("delete expire tombstone failed", err)
+ }
+}
diff --git a/syncer/service/tombstone/tombstone.go b/syncer/service/tombstone/tombstone.go
new file mode 100644
index 0000000..e378bce
--- /dev/null
+++ b/syncer/service/tombstone/tombstone.go
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tombstone
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/apache/servicecomb-service-center/eventbase/model"
+ "github.com/apache/servicecomb-service-center/eventbase/service/tombstone"
+ "github.com/apache/servicecomb-service-center/pkg/log"
+ "github.com/apache/servicecomb-service-center/server/config"
+)
+
+const (
+ defaultExpireTime = time.Hour * 24
+)
+
+func DeleteExpireTombStone() error {
+ expireTime := getExpireTime()
+ request := &model.ListTombstoneRequest{
+ BeforeTimestamp: time.Now().Add(-expireTime).UnixNano(),
+ }
+ tombstones, err := tombstone.List(context.Background(), request)
+ if err != nil {
+ log.Error("get tombstone list fail", err)
+ return err
+ }
+
+ if len(tombstones) <= 0 {
+ log.Info("expire tombstone data is empty")
+ return nil
+ }
+
+ err = tombstone.Delete(context.Background(), tombstones...)
+ if err != nil {
+ log.Error("delete tombstone data list fail", err)
+ return err
+ }
+ return nil
+}
+
+func getExpireTime() time.Duration {
+ reserve := config.GetString("sync.tombstone.retire.reserve", "")
+ if len(reserve) <= 0 {
+ log.Warn("tombstone reserve is empty")
+ return defaultExpireTime
+ }
+
+ expireTime, err := time.ParseDuration(reserve)
+ if err != nil {
+ log.Error(fmt.Sprintf("tombstone expireTime parseDuration expire:%s", reserve), err)
+ return defaultExpireTime
+ }
+ return expireTime
+}
diff --git a/syncer/service/tombstone/tombstone_test.go b/syncer/service/tombstone/tombstone_test.go
new file mode 100644
index 0000000..2f3659e
--- /dev/null
+++ b/syncer/service/tombstone/tombstone_test.go
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tombstone_test
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/apache/servicecomb-service-center/eventbase/datasource"
+ "github.com/apache/servicecomb-service-center/eventbase/model"
+ "github.com/apache/servicecomb-service-center/eventbase/service/tombstone"
+ "github.com/apache/servicecomb-service-center/eventbase/test"
+ synctombstone "github.com/apache/servicecomb-service-center/syncer/service/tombstone"
+ "github.com/go-chassis/cari/sync"
+ "github.com/stretchr/testify/assert"
+)
+
+func init() {
+ err := datasource.Init(test.DbCfg)
+ if err != nil {
+ panic(err)
+ }
+}
+
+const (
+ testDomain = "expireTombstone"
+ testProject = "expireProject"
+)
+
+func TestDeleteExpireTombStone(t *testing.T) {
+ tombstoneOne := sync.NewTombstone(testDomain, testProject, "config", "1")
+ tombstoneOne.Timestamp = time.Now().Add(-time.Hour * 24).UnixNano()
+
+ tombstoneTwo := sync.NewTombstone(testDomain, testProject, "config", "2")
+ tombstoneTwo.Timestamp = time.Now().Add(-time.Hour * 23).UnixNano()
+
+ tombstoneThree := sync.NewTombstone(testDomain, testProject, "config", "3")
+ tombstoneThree.Timestamp = time.Now().Add(-time.Hour * 25).UnixNano()
+
+ t.Run("to create three tasks for next get delete and list operations, should pass", func(t *testing.T) {
+ _, err := datasource.GetDataSource().TombstoneDao().Create(context.Background(), tombstoneOne)
+ assert.Nil(t, err)
+ _, err = datasource.GetDataSource().TombstoneDao().Create(context.Background(), tombstoneTwo)
+ assert.Nil(t, err)
+ _, err = datasource.GetDataSource().TombstoneDao().Create(context.Background(), tombstoneThree)
+ assert.Nil(t, err)
+ })
+
+ t.Run("list tombstone service", func(t *testing.T) {
+ listReq := model.ListTombstoneRequest{
+ Domain: testDomain,
+ Project: testProject,
+ BeforeTimestamp: time.Now().Add(-time.Hour * 24).UnixNano(),
+ }
+ tombstones, err := tombstone.List(context.Background(), &listReq)
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(tombstones))
+ })
+
+ t.Run("delete expire tombstone service", func(t *testing.T) {
+ err := synctombstone.DeleteExpireTombStone()
+ assert.Nil(t, err)
+
+ listReq := model.ListTombstoneRequest{
+ Domain: testDomain,
+ Project: testProject,
+ }
+ tombstones, err := tombstone.List(context.Background(), &listReq)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(tombstones))
+
+ })
+
+ t.Run("delete all tombstones test data", func(t *testing.T) {
+ listReq := model.ListTombstoneRequest{
+ Domain: testDomain,
+ Project: testProject,
+ }
+ tombstones, err := tombstone.List(context.Background(), &listReq)
+ assert.Nil(t, err)
+
+ err = tombstone.Delete(context.Background(), tombstones...)
+ assert.Nil(t, err)
+ })
+
+}