feat: file content and tamplate content add md5 field

bcs-services/bcs-bscp/cmd/config-server/service/app_template_binding.go

@@ -303,6 +303,7 @@ func (s *Service) ListAppBoundTmplRevisions(ctx context.Context, req *pbcs.ListA
 					Creator:              r.Creator,
 					CreateAt:             r.CreateAt,
 					FileState:            r.FileState,
+					Md5:                  r.Md5,
 				})
 		}
 		if req.WithStatus {
@@ -465,6 +466,7 @@ func (s *Service) ListReleasedAppBoundTmplRevisions(ctx context.Context,
 					Reviser:              r.Reviser,
 					CreateAt:             r.CreateAt,
 					UpdateAt:             r.UpdateAt,
+					Md5:                  r.Md5,
 				})
 		}
 		details = append(details, group)

bcs-services/bcs-bscp/cmd/config-server/service/config_item.go

@@ -16,7 +16,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/criteria/errf"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/iam/meta"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/kit"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/logs"
@@ -25,7 +24,6 @@ import (
 	pbci "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/core/config-item"
 	pbcontent "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/core/content"
 	pbds "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/data-service"
-	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/version"
 )
 
 // CreateConfigItem create config item with option
@@ -43,7 +41,8 @@ func (s *Service) CreateConfigItem(ctx context.Context, req *pbcs.CreateConfigIt
 		return nil, err
 	}
 	// 1. validate if file content uploaded.
-	if err = s.validateContentExist(grpcKit, req.BizId, req.Sign); err != nil {
+	metadata, err := s.client.provider.Metadata(grpcKit, req.Sign)
+	if err != nil {
 		logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
 		return nil, err
 	}
@@ -67,6 +66,7 @@ func (s *Service) CreateConfigItem(ctx context.Context, req *pbcs.CreateConfigIt
 		},
 		ContentSpec: &pbcontent.ContentSpec{
 			Signature: req.Sign,
+			Md5:       metadata.Md5,
 			ByteSize:  req.ByteSize,
 		},
 	}
@@ -83,24 +83,6 @@ func (s *Service) CreateConfigItem(ctx context.Context, req *pbcs.CreateConfigIt
 	return resp, nil
 }
 
-func (s *Service) validateContentExist(kt *kit.Kit, _ uint32, sign string) error {
-	// build version is debug mode, not need to validate repo node if exist.
-	if version.Debug() {
-		return nil
-	}
-
-	// validate was file content uploaded.
-	_, err := s.client.provider.Metadata(kt, sign)
-	if err == errf.ErrFileContentNotFound {
-		return fmt.Errorf("file content %s not upload", sign)
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // BatchUpsertConfigItems batch upsert config items with option
 func (s *Service) BatchUpsertConfigItems(ctx context.Context, req *pbcs.BatchUpsertConfigItemsReq) (
 	*pbcs.BatchUpsertConfigItemsResp, error) {
@@ -119,7 +101,8 @@ func (s *Service) BatchUpsertConfigItems(ctx context.Context, req *pbcs.BatchUps
 	items := make([]*pbds.BatchUpsertConfigItemsReq_ConfigItem, 0, len(req.Items))
 	for _, item := range req.Items {
 		// validate if file content uploaded.
-		if err = s.validateContentExist(grpcKit, req.BizId, item.Sign); err != nil {
+		metadata, err := s.client.provider.Metadata(grpcKit, item.Sign)
+		if err != nil {
 			logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
 			return nil, err
 		}
@@ -143,6 +126,7 @@ func (s *Service) BatchUpsertConfigItems(ctx context.Context, req *pbcs.BatchUps
 			ContentSpec: &pbcontent.ContentSpec{
 				Signature: item.Sign,
 				ByteSize:  item.ByteSize,
+				Md5:       metadata.Md5,
 			},
 		})
 	}
@@ -223,6 +207,12 @@ func (s *Service) UpdateConfigItem(ctx context.Context, req *pbcs.UpdateConfigIt
 	}
 
 	// 2.3 if latest content sign not equals request content sign,create content and commit
+	// validate if file content uploaded.
+	metadata, err := s.client.provider.Metadata(grpcKit, req.Sign)
+	if err != nil {
+		logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
+		return nil, err
+	}
 	ccReq := &pbds.CreateContentReq{
 		Attachment: &pbcontent.ContentAttachment{
 			ConfigItemId: req.Id,
@@ -232,13 +222,9 @@ func (s *Service) UpdateConfigItem(ctx context.Context, req *pbcs.UpdateConfigIt
 		Spec: &pbcontent.ContentSpec{
 			Signature: req.Sign,
 			ByteSize:  req.ByteSize,
+			Md5:       metadata.Md5,
 		},
 	}
-	// validate if file content uploaded.
-	if err = s.validateContentExist(grpcKit, req.BizId, req.Sign); err != nil {
-		logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
-		return nil, err
-	}
 	ccResp, err := s.client.DS.CreateContent(grpcKit.RpcCtx(), ccReq)
 	if err != nil {
 		logs.Errorf("create config item failed, err: %v, rid: %s", err, grpcKit.Rid)

bcs-services/bcs-bscp/cmd/config-server/service/template.go

@@ -15,6 +15,7 @@ package service
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"golang.org/x/sync/errgroup"
 
@@ -52,6 +53,13 @@ func (s *Service) CreateTemplate(ctx context.Context, req *pbcs.CreateTemplateRe
 			idsLen, constant.ArrayInputLenLimit)
 	}
 
+	// validate if file content uploaded.
+	metadata, err := s.client.provider.Metadata(grpcKit, req.Sign)
+	if err != nil {
+		logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
+		return nil, err
+	}
+
 	r := &pbds.CreateTemplateReq{
 		Attachment: &pbtemplate.TemplateAttachment{
 			BizId:           grpcKit.BizID,
@@ -76,6 +84,7 @@ func (s *Service) CreateTemplate(ctx context.Context, req *pbcs.CreateTemplateRe
 			ContentSpec: &pbcontent.ContentSpec{
 				Signature: req.Sign,
 				ByteSize:  req.ByteSize,
+				Md5:       metadata.Md5,
 			},
 		},
 		TemplateSetIds: req.TemplateSetIds,
@@ -467,60 +476,63 @@ func (s *Service) BatchUpsertTemplates(ctx context.Context, req *pbcs.BatchUpser
 	if err := s.authorizer.Authorize(grpcKit, res...); err != nil {
 		return nil, err
 	}
+	items := make([]*pbds.BatchUpsertTemplatesReq_Item, 0, len(req.Items))
+	var mu sync.Mutex
 	var g errgroup.Group
 	g.SetLimit(constant.MaxConcurrentUpload)
 	for _, item := range req.Items {
-		sign := item.Sign
+		i := item
 		g.Go(func() error {
 			// validate if file content uploaded.
-			if err := s.validateContentExist(grpcKit, req.BizId, sign); err != nil {
+			metadata, err := s.client.provider.Metadata(grpcKit, i.Sign)
+			if err != nil {
 				logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
 				return err
 			}
-			return nil
-		})
-	}
-	if err := g.Wait(); err != nil {
-		return nil, err
-	}
-	items := make([]*pbds.BatchUpsertTemplatesReq_Item, 0, len(req.Items))
-	for _, item := range req.Items {
-		items = append(items, &pbds.BatchUpsertTemplatesReq_Item{
-			Template: &pbtemplate.Template{
-				Id: item.Id,
-				Spec: &pbtemplate.TemplateSpec{
-					Name: item.Name,
-					Path: item.Path,
-					Memo: item.Memo,
-				},
-				Attachment: &pbtemplate.TemplateAttachment{
-					BizId:           req.BizId,
-					TemplateSpaceId: req.TemplateSpaceId,
-				},
-			},
-			TemplateRevision: &pbtr.TemplateRevision{
-				Spec: &pbtr.TemplateRevisionSpec{
-					Name:     item.Name,
-					Path:     item.Path,
-					FileType: item.FileType,
-					FileMode: item.FileMode,
-					Permission: &pbci.FilePermission{
-						User:      item.User,
-						UserGroup: item.UserGroup,
-						Privilege: item.Privilege,
+			mu.Lock()
+			items = append(items, &pbds.BatchUpsertTemplatesReq_Item{
+				Template: &pbtemplate.Template{
+					Id: i.Id,
+					Spec: &pbtemplate.TemplateSpec{
+						Name: i.Name,
+						Path: i.Path,
+						Memo: i.Memo,
 					},
-					ContentSpec: &pbcontent.ContentSpec{
-						Signature: item.Sign,
-						ByteSize:  item.ByteSize,
+					Attachment: &pbtemplate.TemplateAttachment{
+						BizId:           req.BizId,
+						TemplateSpaceId: req.TemplateSpaceId,
 					},
 				},
-				Attachment: &pbtr.TemplateRevisionAttachment{
-					BizId:           req.BizId,
-					TemplateSpaceId: req.TemplateSpaceId,
+				TemplateRevision: &pbtr.TemplateRevision{
+					Spec: &pbtr.TemplateRevisionSpec{
+						Name:     i.Name,
+						Path:     i.Path,
+						FileType: i.FileType,
+						FileMode: i.FileMode,
+						Permission: &pbci.FilePermission{
+							User:      i.User,
+							UserGroup: i.UserGroup,
+							Privilege: i.Privilege,
+						},
+						ContentSpec: &pbcontent.ContentSpec{
+							Signature: i.Sign,
+							ByteSize:  i.ByteSize,
+							Md5:       metadata.Md5,
+						},
+					},
+					Attachment: &pbtr.TemplateRevisionAttachment{
+						BizId:           req.BizId,
+						TemplateSpaceId: req.TemplateSpaceId,
+					},
 				},
-			},
+			})
+			mu.Unlock()
+			return nil
 		})
 	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
 	in := &pbds.BatchUpsertTemplatesReq{Items: items}
 	data, err := s.client.DS.BatchUpsertTemplates(grpcKit.RpcCtx(), in)
 	if err != nil {

bcs-services/bcs-bscp/cmd/config-server/service/template_revision.go

@@ -40,6 +40,12 @@ func (s *Service) CreateTemplateRevision(ctx context.Context, req *pbcs.CreateTe
 		return nil, err
 	}
 
+	metadata, err := s.client.provider.Metadata(grpcKit, req.Sign)
+	if err != nil {
+		logs.Errorf("validate file content uploaded failed, err: %v, rid: %s", err, grpcKit.Rid)
+		return nil, err
+	}
+
 	r := &pbds.CreateTemplateRevisionReq{
 		Attachment: &pbtr.TemplateRevisionAttachment{
 			BizId:           grpcKit.BizID,
@@ -61,6 +67,7 @@ func (s *Service) CreateTemplateRevision(ctx context.Context, req *pbcs.CreateTe
 			ContentSpec: &pbcontent.ContentSpec{
 				Signature: req.Sign,
 				ByteSize:  req.ByteSize,
+				Md5:       metadata.Md5,
 			},
 		},
 	}