You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2018/01/22 13:22:19 UTC

[incubator-servicecomb-service-center] branch master updated: SCB-269 dependency use domain-project-env level lock (#253)

This is an automated email from the ASF dual-hosted git repository.

littlecui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-servicecomb-service-center.git


The following commit(s) were added to refs/heads/master by this push:
     new 64d1feb  SCB-269 dependency use domain-project-env level lock (#253)
64d1feb is described below

commit 64d1febae37681af891ac8126eff771382ed81ce
Author: aseTo2016 <14...@qq.com>
AuthorDate: Mon Jan 22 21:22:17 2018 +0800

    SCB-269 dependency use domain-project-env level lock (#253)
    
    * [scb255] dependency use domain-project-env level lock
    
    (cherry picked from commit 9ee70e0)
    
    * [scb255] dependency use domain-project-env level lock
    
    * [scb255] dependency use domain-project-env level lock
    
    * [SCB-269] dependency use domain-project-env level lock
---
 server/service/event/dependency_event_handler.go |  69 +++++++++++-
 server/service/microservices.go                  |  83 ++++++++-------
 server/service/util/dependency.go                | 127 +++++++----------------
 server/service/util/dependency_test.go           |  16 +--
 4 files changed, 152 insertions(+), 143 deletions(-)

diff --git a/server/service/event/dependency_event_handler.go b/server/service/event/dependency_event_handler.go
index 6789a14..c91cb35 100644
--- a/server/service/event/dependency_event_handler.go
+++ b/server/service/event/dependency_event_handler.go
@@ -102,6 +102,20 @@ func (h *DependencyEventHandler) loop() {
 	})
 }
 
+type DependencyEventHandlerResource struct {
+	dep    *pb.ConsumerDependency
+	kv     *mvccpb.KeyValue
+	domainProject  string
+}
+
+func NewDependencyEventHandlerResource(dep *pb.ConsumerDependency, kv *mvccpb.KeyValue, domainProject string) *DependencyEventHandlerResource {
+	return &DependencyEventHandlerResource{
+		dep,
+		kv,
+		domainProject,
+	}
+}
+
 func (h *DependencyEventHandler) Handle() error {
 	key := core.GetServiceDependencyQueueRootKey("")
 	resp, err := store.Store().DependencyQueue().Search(context.Background(),
@@ -117,6 +131,9 @@ func (h *DependencyEventHandler) Handle() error {
 		return nil
 	}
 
+	lenKvs := len(resp.Kvs)
+	resourcesMap := make(map[string][]*DependencyEventHandlerResource, lenKvs)
+
 	ctx := context.Background()
 	for _, kv := range resp.Kvs {
 		r := &pb.ConsumerDependency{}
@@ -133,19 +150,63 @@ func (h *DependencyEventHandler) Handle() error {
 			continue
 		}
 
+		lockKey := serviceUtil.NewDependencyLockKey(domainProject, r.Consumer.Environment)
+		res := NewDependencyEventHandlerResource(r, kv, domainProject)
+		resources := resourcesMap[lockKey]
+		if resources == nil {
+			resources = make([]*DependencyEventHandlerResource, 0, lenKvs)
+		}
+		resources = append(resources, res)
+		resourcesMap[lockKey] = resources
+	}
+
+	dependencyRuleHandleResults := make(chan error, len(resourcesMap))
+	for lockKey, resources := range resourcesMap {
+		go func(lockKey string, resources []*DependencyEventHandlerResource){
+			err := h.dependencyRuleHandle(ctx, lockKey, resources)
+			dependencyRuleHandleResults <- err
+		}(lockKey, resources)
+	}
+	var lastErr error
+	finishedCount := 0
+	for err := range dependencyRuleHandleResults {
+		finishedCount++
+		if err != nil {
+			lastErr = err
+		}
+		if finishedCount == len(resourcesMap) {
+			close(dependencyRuleHandleResults)
+		}
+	}
+	return lastErr
+}
+
+func (h *DependencyEventHandler)dependencyRuleHandle(ctx context.Context, lockKey string, resources []*DependencyEventHandlerResource) error{
+	lock, err := serviceUtil.DependencyLock(lockKey)
+	if err != nil {
+		util.Logger().Errorf(err, "create dependency rule locker failed")
+		return err
+	}
+	defer lock.Unlock()
+	for _, res := range resources {
+		r := res.dep
 		consumerFlag := util.StringJoin([]string{r.Consumer.AppId, r.Consumer.ServiceName, r.Consumer.Version}, "/")
+
+		domainProject := res.domainProject
 		consumerInfo := pb.DependenciesToKeys([]*pb.MicroServiceKey{r.Consumer}, domainProject)[0]
 		providersInfo := pb.DependenciesToKeys(r.Providers, domainProject)
 
-		consumerId, err = serviceUtil.GetServiceId(ctx, consumerInfo)
+		consumerId, err := serviceUtil.GetServiceId(ctx, consumerInfo)
 		if err != nil {
+			util.Logger().Errorf(err, "modify dependency rule failed, override: %t, consumer %s", r.Override, consumerFlag)
 			return fmt.Errorf("get consumer %s id failed, override: %t, %s", consumerFlag, r.Override, err.Error())
 		}
 		if len(consumerId) == 0 {
 			util.Logger().Errorf(nil, "maintain dependency failed, override: %t, consumer %s does not exist",
 				r.Override, consumerFlag)
 
-			if err = h.removeKV(ctx, kv); err != nil {
+			if err = h.removeKV(ctx, res.kv); err != nil {
+				util.Logger().Errorf(err, "remove dependency rule failed, override: %t, consumer %s", r.Override, consumerFlag)
 				return err
 			}
 			continue
@@ -163,10 +224,12 @@ func (h *DependencyEventHandler) Handle() error {
 		}
 
 		if err != nil {
+			util.Logger().Errorf(err, "modify dependency rule failed, override: %t, consumer %s", r.Override, consumerFlag)
 			return fmt.Errorf("override: %t, consumer is %s, %s", r.Override, consumerFlag, err.Error())
 		}
 
-		if err = h.removeKV(ctx, kv); err != nil {
+		if err = h.removeKV(ctx, res.kv); err != nil {
+			util.Logger().Errorf(err, "remove dependency rule failed, override: %t, consumer %s", r.Override, consumerFlag)
 			return err
 		}
 
diff --git a/server/service/microservices.go b/server/service/microservices.go
index 578c408..319c108 100644
--- a/server/service/microservices.go
+++ b/server/service/microservices.go
@@ -28,7 +28,6 @@ import (
 	scerr "github.com/apache/incubator-servicecomb-service-center/server/error"
 	"github.com/apache/incubator-servicecomb-service-center/server/infra/quota"
 	"github.com/apache/incubator-servicecomb-service-center/server/infra/registry"
-	"github.com/apache/incubator-servicecomb-service-center/server/mux"
 	"github.com/apache/incubator-servicecomb-service-center/server/plugin"
 	serviceUtil "github.com/apache/incubator-servicecomb-service-center/server/service/util"
 	"golang.org/x/net/context"
@@ -196,7 +195,7 @@ func checkQuota(ctx context.Context, domainProject string) (quota.QuotaReporter,
 	return rst.Reporter, rst.Err
 }
 
-func (s *MicroServiceService) DeleteServicePri(ctx context.Context, ServiceId string, force bool) (*pb.Response, error) {
+func (s *MicroServiceService) DeleteServicePri(ctx context.Context, serviceId string, force bool) (*pb.Response, error) {
 	domainProject := util.ParseDomainProject(ctx)
 
 	title := "delete"
@@ -204,20 +203,24 @@ func (s *MicroServiceService) DeleteServicePri(ctx context.Context, ServiceId st
 		title = "force delete"
 	}
 
-	if ServiceId == apt.Service.ServiceId {
+	isServiceCenter := func(serviceId string) bool {
+		return serviceId == apt.Service.ServiceId
+	}
+
+	if isServiceCenter(serviceId) {
 		err := errors.New("not allow to delete service center")
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s", title, serviceId)
 		return pb.CreateResponse(scerr.ErrInvalidParams, err.Error()), nil
 	}
 
-	service, err := serviceUtil.GetService(ctx, domainProject, ServiceId)
+	service, err := serviceUtil.GetService(ctx, domainProject, serviceId)
 	if err != nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: get service failed.", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: get service failed.", title, serviceId)
 		return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
 	}
 
 	if service == nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: service not exist.", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: service not exist.", title, serviceId)
 		return pb.CreateResponse(scerr.ErrServiceNotExists, "Service does not exist."), nil
 	}
 
@@ -226,31 +229,38 @@ func (s *MicroServiceService) DeleteServicePri(ctx context.Context, ServiceId st
 		dr := serviceUtil.NewProviderDependencyRelation(ctx, domainProject, service)
 		services, err := dr.GetDependencyConsumerIds()
 		if err != nil {
-			util.Logger().Errorf(err, "delete micro-service failed, serviceId is %s: inner err, get service dependency failed.", ServiceId)
+			util.Logger().Errorf(err, "delete micro-service failed, serviceId is %s: inner err, get service dependency failed.", serviceId)
 			return pb.CreateResponse(scerr.ErrInternal, "Get dependency info failed."), err
 		}
-		if len(services) > 1 || (len(services) == 1 && services[0] != ServiceId) {
-			util.Logger().Errorf(nil, "delete micro-service failed, serviceId is %s: can't delete, other services rely it.", ServiceId)
+		if len(services) > 1 || (len(services) == 1 && services[0] != serviceId) {
+			util.Logger().Errorf(nil, "delete micro-service failed, serviceId is %s: can't delete, other services rely it.", serviceId)
 			return pb.CreateResponse(scerr.ErrDependedOnConsumer, "Can not delete this service, other service rely it."), err
 		}
 
-		instancesKey := apt.GenerateInstanceKey(domainProject, ServiceId, "")
+		instancesKey := apt.GenerateInstanceKey(domainProject, serviceId, "")
 		rsp, err := store.Store().Instance().Search(ctx,
 			registry.WithStrKey(instancesKey),
 			registry.WithPrefix(),
 			registry.WithCountOnly())
 		if err != nil {
-			util.Logger().Errorf(err, "delete micro-service failed, serviceId is %s: inner err,get instances failed.", ServiceId)
+			util.Logger().Errorf(err, "delete micro-service failed, serviceId is %s: inner err,get instances failed.", serviceId)
 			return pb.CreateResponse(scerr.ErrInternal, "Get instance failed."), err
 		}
 
 		if rsp.Count > 0 {
-			util.Logger().Errorf(nil, "delete micro-service failed, serviceId is %s: can't delete, exist instance.", ServiceId)
+			util.Logger().Errorf(nil, "delete micro-service failed, serviceId is %s: can't delete, exist instance.", serviceId)
 			return pb.CreateResponse(scerr.ErrDeployedInstance, "Can not delete this service, exist instance."), err
 		}
 	}
 
-	consumer := &pb.MicroServiceKey{
+	//refresh msCache consumerCache, ensure that watch can notify consumers when no cache.
+	err = serviceUtil.RefreshDependencyCache(ctx, domainProject, service)
+	if err != nil {
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: inner err, refresh service dependency cache failed.", title, serviceId)
+		return pb.CreateResponse(scerr.ErrInternal, "Refresh dependency cache failed."), err
+	}
+
+	serviceKey := &pb.MicroServiceKey{
 		Tenant:      domainProject,
 		Environment: service.Environment,
 		AppId:       service.AppId,
@@ -258,71 +268,64 @@ func (s *MicroServiceService) DeleteServicePri(ctx context.Context, ServiceId st
 		Version:     service.Version,
 		Alias:       service.Alias,
 	}
-
-	//refresh msCache consumerCache, ensure that watch can notify consumers when no cache.
-	err = serviceUtil.RefreshDependencyCache(ctx, domainProject, service)
-	if err != nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: inner err, refresh service dependency cache failed.", title, ServiceId)
-		return pb.CreateResponse(scerr.ErrInternal, "Refresh dependency cache failed."), err
-	}
-
 	opts := []registry.PluginOp{
-		registry.OpDel(registry.WithStrKey(apt.GenerateServiceIndexKey(consumer))),
-		registry.OpDel(registry.WithStrKey(apt.GenerateServiceAliasKey(consumer))),
-		registry.OpDel(registry.WithStrKey(apt.GenerateServiceKey(domainProject, ServiceId))),
+		registry.OpDel(registry.WithStrKey(apt.GenerateServiceIndexKey(serviceKey))),
+		registry.OpDel(registry.WithStrKey(apt.GenerateServiceAliasKey(serviceKey))),
+		registry.OpDel(registry.WithStrKey(apt.GenerateServiceKey(domainProject, serviceId))),
 		registry.OpDel(registry.WithStrKey(
-			util.StringJoin([]string{apt.GetServiceRuleRootKey(domainProject), ServiceId, ""}, "/"))),
+			util.StringJoin([]string{apt.GetServiceRuleRootKey(domainProject), serviceId, ""}, "/"))),
 	}
 
 	//删除依赖规则
-	lock, err := mux.Lock(mux.DEP_QUEUE_LOCK)
+	lock, err := serviceUtil.DependencyLock(serviceUtil.NewDependencyLockKey(domainProject, service.Environment))
 	if err != nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: inner err, create lock failed.", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: inner err, create lock failed.", title, serviceId)
 		return pb.CreateResponse(scerr.ErrUnavailableBackend, err.Error()), err
 	}
-	optsTmp, err := serviceUtil.DeleteDependencyForService(ctx, consumer, ServiceId)
-	lock.Unlock()
+
+	defer lock.Unlock()
+	optsTmp, err := serviceUtil.DeleteDependencyForService(ctx, serviceKey)
 	if err != nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: inner err, delete dependency failed.", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: inner err, delete dependency failed.", title, serviceId)
 		return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
 	}
 	opts = append(opts, optsTmp...)
 
 	//删除黑白名单
 	opts = append(opts, registry.OpDel(
-		registry.WithStrKey(apt.GenerateServiceRuleKey(domainProject, ServiceId, "")),
+		registry.WithStrKey(apt.GenerateServiceRuleKey(domainProject, serviceId, "")),
 		registry.WithPrefix()))
 	opts = append(opts, registry.OpDel(
-		registry.WithStrKey(apt.GenerateRuleIndexKey(domainProject, ServiceId, "", ""))))
+		registry.WithStrKey(apt.GenerateRuleIndexKey(domainProject, serviceId, "", ""))))
 
 	//删除schemas
 	opts = append(opts, registry.OpDel(
-		registry.WithStrKey(apt.GenerateServiceSchemaKey(domainProject, ServiceId, "")),
+		registry.WithStrKey(apt.GenerateServiceSchemaKey(domainProject, serviceId, "")),
 		registry.WithPrefix()))
 	opts = append(opts, registry.OpDel(
-		registry.WithStrKey(apt.GenerateServiceSchemaSummaryKey(domainProject, ServiceId, "")),
+		registry.WithStrKey(apt.GenerateServiceSchemaSummaryKey(domainProject, serviceId, "")),
 		registry.WithPrefix()))
 
 	//删除tags
 	opts = append(opts, registry.OpDel(
-		registry.WithStrKey(apt.GenerateServiceTagKey(domainProject, ServiceId))))
+		registry.WithStrKey(apt.GenerateServiceTagKey(domainProject, serviceId))))
 
 	//删除实例
-	err = serviceUtil.DeleteServiceAllInstances(ctx, ServiceId)
+	err = serviceUtil.DeleteServiceAllInstances(ctx, serviceId)
 	if err != nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: delete all instances failed.", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: delete all instances failed.", title, serviceId)
 		return pb.CreateResponse(scerr.ErrInternal, "Delete all instances failed for service."), err
 	}
 
 	err = backend.BatchCommit(ctx, opts)
 	if err != nil {
-		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: commit data into etcd failed.", title, ServiceId)
+		util.Logger().Errorf(err, "%s micro-service failed, serviceId is %s: commit data into etcd failed.", title, serviceId)
 		return pb.CreateResponse(scerr.ErrUnavailableBackend, "Commit operations failed."), nil
 	}
 
 	serviceUtil.RemandServiceQuota(ctx)
 
-	util.Logger().Infof("%s micro-service successful: serviceId is %s, operator is %s.", title, ServiceId, util.GetIPFromContext(ctx))
+	util.Logger().Infof("%s micro-service successful: serviceId is %s, operator is %s.", title, serviceId, util.GetIPFromContext(ctx))
 	return pb.CreateResponse(pb.Response_SUCCESS, "Unregister service successfully."), nil
 }
 
diff --git a/server/service/util/dependency.go b/server/service/util/dependency.go
index bd4a3ac..5bfb978 100644
--- a/server/service/util/dependency.go
+++ b/server/service/util/dependency.go
@@ -31,6 +31,8 @@ import (
 	"golang.org/x/net/context"
 	"strings"
 	"time"
+	"github.com/apache/incubator-servicecomb-service-center/pkg/etcdsync"
+	"github.com/apache/incubator-servicecomb-service-center/server/mux"
 )
 
 var consumerCache *cache.Cache
@@ -277,54 +279,40 @@ func AddServiceVersionRule(ctx context.Context, domainProject string, consumer *
 	return nil
 }
 
-func DeleteDependencyForService(ctx context.Context, consumer *pb.MicroServiceKey, serviceId string) ([]registry.PluginOp, error) {
-	ops := []registry.PluginOp{}
-	opsTmps := []registry.PluginOp{}
-	domainProject := consumer.Tenant
-	flag := map[string]bool{}
+func DeleteDependencyForService(ctx context.Context, service *pb.MicroServiceKey) ([]registry.PluginOp, error) {
+	domainProject := service.Tenant
 	//删除依赖规则
-	conKey := apt.GenerateConsumerDependencyRuleKey(domainProject, consumer)
+	conKey := apt.GenerateConsumerDependencyRuleKey(domainProject, service)
 	providerValue, err := TransferToMicroServiceDependency(ctx, conKey)
 	if err != nil {
 		return nil, err
 	}
+	opts := make([]registry.PluginOp, 0)
 	if providerValue != nil && len(providerValue.Dependency) != 0 {
-		proProkey := ""
+		providerRuleKey := ""
 		for _, providerRule := range providerValue.Dependency {
-			proProkey = apt.GenerateProviderDependencyRuleKey(domainProject, providerRule)
-			consumers, err := TransferToMicroServiceDependency(ctx, proProkey)
+			providerRuleKey = apt.GenerateProviderDependencyRuleKey(domainProject, providerRule)
+			consumers, err := TransferToMicroServiceDependency(ctx, providerRuleKey)
 			if err != nil {
 				return nil, err
 			}
-			err = deleteDependencyRuleUtil(ctx, consumers, consumer, proProkey)
+			opt, err := updateProviderDependencyRuleUtil(consumers, service, providerRuleKey)
 			if err != nil {
 				return nil, err
 			}
+			opts = append(opts, opt)
 		}
-
-		util.Logger().Debugf("conKey is %s.", conKey)
-		ops = append(ops, registry.OpDel(registry.WithStrKey(conKey)))
 	}
+	util.Logger().Infof("delete dependency rule, consumer Key is %s.", conKey)
+	opts = append(opts, registry.OpDel(registry.WithStrKey(conKey)))
+
 	//作为provider的依赖规则
-	providerKey := apt.GenerateProviderDependencyRuleKey(domainProject, consumer)
+	providerKey := apt.GenerateProviderDependencyRuleKey(domainProject, service)
 
-	util.Logger().Debugf("providerKey is %s", providerKey)
-	ops = append(ops, registry.OpDel(registry.WithStrKey(providerKey)))
+	util.Logger().Infof("delete dependency rule, providerKey is %s", providerKey)
+	opts = append(opts, registry.OpDel(registry.WithStrKey(providerKey)))
 
-	//删除依赖关系
-	opsTmps, err = deleteDependencyUtil(ctx, "c", domainProject, serviceId, flag)
-	if err != nil {
-		return nil, err
-	}
-	ops = append(ops, opsTmps...)
-	util.Logger().Debugf("flag is %s", flag)
-	opsTmps, err = deleteDependencyUtil(ctx, "p", domainProject, serviceId, flag)
-	if err != nil {
-		return nil, err
-	}
-	util.Logger().Debugf("flag is %s", flag)
-	ops = append(ops, opsTmps...)
-	return ops, nil
+	return opts, nil
 }
 
 func TransferToMicroServiceDependency(ctx context.Context, key string) (*pb.MicroServiceDependency, error) {
@@ -348,35 +336,25 @@ func TransferToMicroServiceDependency(ctx context.Context, key string) (*pb.Micr
 	return microServiceDependency, nil
 }
 
-func deleteDependencyRuleUtil(ctx context.Context, microServiceDependency *pb.MicroServiceDependency, service *pb.MicroServiceKey, serviceKey string) error {
-	for key, serviceTmp := range microServiceDependency.Dependency {
-		if ok := equalServiceDependency(serviceTmp, service); ok {
-			microServiceDependency.Dependency = append(microServiceDependency.Dependency[:key], microServiceDependency.Dependency[key+1:]...)
-			util.Logger().Debugf("delete versionRule from %s", serviceTmp.ServiceName)
+func updateProviderDependencyRuleUtil(consumersOfProvideRules *pb.MicroServiceDependency, consumer *pb.MicroServiceKey, providerRuleKey string) (registry.PluginOp, error) {
+	for key, consumerInner := range consumersOfProvideRules.Dependency {
+		if ok := equalServiceDependency(consumerInner, consumer); ok {
+			consumersOfProvideRules.Dependency = append(consumersOfProvideRules.Dependency[:key], consumersOfProvideRules.Dependency[key+1:]...)
 			break
 		}
 	}
-	opts := []registry.PluginOpOption{}
-	if len(microServiceDependency.Dependency) == 0 {
-		opts = append(opts, registry.DEL, registry.WithStrKey(serviceKey))
-		util.Logger().Debugf("serviceKey is .", serviceKey)
-		util.Logger().Debugf("After deleting versionRule from %s,provider's consumer is empty.", serviceKey)
-
+	if len(consumersOfProvideRules.Dependency) == 0 {
+		util.Logger().Infof("delete dependency rule key is %s", providerRuleKey)
+		return registry.OpDel(registry.WithStrKey(providerRuleKey)), nil
 	} else {
-		data, err := json.Marshal(microServiceDependency)
+		data, err := json.Marshal(consumersOfProvideRules)
 		if err != nil {
-			util.Logger().Errorf(nil, "Marshal tmpValue failed.")
-			return err
+			util.Logger().Errorf(nil, "update dependency key Marshal tmpValue failed.")
+			return registry.PluginOp{}, err
 		}
-		opts = append(opts, registry.PUT, registry.WithStrKey(serviceKey), registry.WithValue(data))
-		util.Logger().Debugf("serviceKey is %s.", serviceKey)
+		util.Logger().Infof("put provider's dependency rule, key is  %s, value is %v.", providerRuleKey, consumersOfProvideRules)
+		return registry.OpPut(registry.WithStrKey(providerRuleKey), registry.WithValue(data)), nil
 	}
-	_, err := backend.Registry().Do(ctx, opts...)
-	if err != nil {
-		util.Logger().Errorf(err, "Submit update dependency failed.")
-		return err
-	}
-	return nil
 }
 
 func equalServiceDependency(serviceA *pb.MicroServiceKey, serviceB *pb.MicroServiceKey) bool {
@@ -402,43 +380,6 @@ func toString(in *pb.MicroServiceKey) string {
 	return apt.GenerateProviderDependencyRuleKey(in.Tenant, in)
 }
 
-func deleteDependencyUtil(ctx context.Context, serviceType string, domainProject string, serviceId string, flag map[string]bool) ([]registry.PluginOp, error) {
-	serviceKey := apt.GenerateServiceDependencyKey(serviceType, domainProject, serviceId, "")
-	rsp, err := store.Store().Dependency().Search(ctx,
-		registry.WithStrKey(serviceKey),
-		registry.WithPrefix())
-	if err != nil {
-		return nil, err
-	}
-	ops := []registry.PluginOp{}
-	if rsp != nil {
-		serviceTmpId := ""
-		serviceTmpKey := ""
-		deleteKey := ""
-		for _, kv := range rsp.Kvs {
-			tmpKeyArr := strings.Split(util.BytesToStringWithNoCopy(kv.Key), "/")
-			serviceTmpId = tmpKeyArr[len(tmpKeyArr)-1]
-			if serviceType == "p" {
-				serviceTmpKey = apt.GenerateConsumerDependencyKey(domainProject, serviceTmpId, serviceId)
-				deleteKey = util.StringJoin([]string{"c", serviceTmpId, serviceId}, "/")
-			} else {
-				serviceTmpKey = apt.GenerateProviderDependencyKey(domainProject, serviceTmpId, serviceId)
-				deleteKey = util.StringJoin([]string{"p", serviceTmpId, serviceId}, "/")
-			}
-			if _, ok := flag[serviceTmpKey]; ok {
-				util.Logger().Debugf("serviceTmpKey is more exist.%s", serviceTmpKey)
-				continue
-			}
-			flag[serviceTmpKey] = true
-			util.Logger().Infof("delete dependency %s", deleteKey)
-			ops = append(ops, registry.OpDel(registry.WithStrKey(serviceTmpKey)))
-		}
-		util.Logger().Infof("delete dependency serviceKey is %s", serviceType+"/"+serviceId)
-		ops = append(ops, registry.OpDel(registry.WithStrKey(serviceKey), registry.WithPrefix()))
-	}
-	return ops, nil
-}
-
 func parseAddOrUpdateRules(ctx context.Context, dep *Dependency) (newDependencyRuleList, existDependencyRuleList, deleteDependencyRuleList []*pb.MicroServiceKey) {
 	conKey := apt.GenerateConsumerDependencyRuleKey(dep.DomainProject, dep.Consumer)
 
@@ -1112,3 +1053,11 @@ func (dr *DependencyRelation) getConsumerOfSameServiceNameAndAppId(provider *pb.
 	}
 	return allConsumers, nil
 }
+
+func DependencyLock(lockKey string) (*etcdsync.DLock, error) {
+	return mux.Lock(mux.MuxType(lockKey))
+}
+
+func NewDependencyLockKey(domainProject, env string) string {
+	return util.StringJoin([]string{"","env-lock", domainProject, env}, "/")
+}
\ No newline at end of file
diff --git a/server/service/util/dependency_test.go b/server/service/util/dependency_test.go
index 6695945..c1326e9 100644
--- a/server/service/util/dependency_test.go
+++ b/server/service/util/dependency_test.go
@@ -33,13 +33,13 @@ func TestRefreshDependencyCache(t *testing.T) {
 }
 
 func TestDeleteDependencyForService(t *testing.T) {
-	_, err := DeleteDependencyForService(context.Background(), &proto.MicroServiceKey{}, "")
+	_, err := DeleteDependencyForService(context.Background(), &proto.MicroServiceKey{})
 	if err == nil {
 		fmt.Printf(`DeleteDependencyForService failed`)
 		t.FailNow()
 	}
 
-	err = deleteDependencyRuleUtil(context.Background(),
+	_, err = updateProviderDependencyRuleUtil(
 		&proto.MicroServiceDependency{
 			Dependency: []*proto.MicroServiceKey{
 				{AppId: "a"},
@@ -48,12 +48,12 @@ func TestDeleteDependencyForService(t *testing.T) {
 		&proto.MicroServiceKey{
 			AppId: "a",
 		}, "")
-	if err == nil {
+	if err != nil {
 		fmt.Printf(`deleteDependencyRuleUtil with the same deps failed`)
 		t.FailNow()
 	}
 
-	err = deleteDependencyRuleUtil(context.Background(),
+	_, err = updateProviderDependencyRuleUtil(
 		&proto.MicroServiceDependency{
 			Dependency: []*proto.MicroServiceKey{
 				{AppId: "b"},
@@ -62,17 +62,11 @@ func TestDeleteDependencyForService(t *testing.T) {
 		&proto.MicroServiceKey{
 			AppId: "a",
 		}, "")
-	if err == nil {
+	if err != nil {
 		fmt.Printf(`deleteDependencyRuleUtil failed`)
 		t.FailNow()
 	}
 
-	_, err = deleteDependencyUtil(context.Background(), "", "", "", map[string]bool{})
-	if err == nil {
-		fmt.Printf(`deleteDependencyUtil failed`)
-		t.FailNow()
-	}
-
 	_, err = deleteConsumerDepOfProviderRule(context.Background(), "", &proto.MicroServiceKey{}, &proto.MicroServiceKey{})
 	if err == nil {
 		fmt.Printf(`deleteConsumerDepOfProviderRule failed`)

-- 
To stop receiving notification emails like this one, please contact
littlecui@apache.org.