From 881697226a0469dc15b33ac9d94d482fbe1b70d1 Mon Sep 17 00:00:00 2001 From: Naoki MATSUMOTO Date: Fri, 29 Mar 2024 19:31:01 +0900 Subject: [PATCH] change DimgHeader format Signed-off-by: Naoki MATSUMOTO --- cmd/ctr-cli/load/load.go | 6 +- cmd/ctr-cli/main.go | 6 +- cmd/ctr-cli/merge/merge.go | 140 +++++++++++++++++++++++++++++-------- cmd/ctr-cli/patch/patch.go | 4 +- cmd/ctr-cli/show/show.go | 87 +++++++++++++++++++++++ cmd/fuse-diff/main.go | 68 +++++++++--------- cmd/server/main.go | 27 ++++++- pkg/di3fs/di3fs.go | 21 +++--- pkg/image/cdimg.go | 38 +++++----- pkg/image/diff.go | 34 +++------ pkg/image/dimg.go | 6 +- pkg/image/merge.go | 52 +++++++++++++- pkg/image/pack.go | 7 +- tests/.gitignore | 1 + tests/bench_impl.sh | 6 +- tests/bench_patch_impl.sh | 2 +- 16 files changed, 378 insertions(+), 127 deletions(-) create mode 100644 cmd/ctr-cli/show/show.go diff --git a/cmd/ctr-cli/load/load.go b/cmd/ctr-cli/load/load.go index 92cb8b7..4884343 100644 --- a/cmd/ctr-cli/load/load.go +++ b/cmd/ctr-cli/load/load.go @@ -27,8 +27,8 @@ var Flags = []cli.Flag{ Required: true, }, &cli.StringFlag{ - Name: "dimg", - Usage: "path to dimg to be loaded", + Name: "cdimg", + Usage: "path to cdimg to be loaded", Required: true, }, } @@ -150,7 +150,7 @@ func Load(ctx context.Context, imgNameWithVersion, imgPath string) error { func Action(c *cli.Context) error { imgName := c.String("image") - imgPath := c.String("dimg") + imgPath := c.String("cdimg") err := Load(context.TODO(), imgName, imgPath) if err != nil { return err diff --git a/cmd/ctr-cli/main.go b/cmd/ctr-cli/main.go index 4fa9d5b..d8580b6 100644 --- a/cmd/ctr-cli/main.go +++ b/cmd/ctr-cli/main.go @@ -11,6 +11,7 @@ import ( "github.com/naoki9911/fuse-diff-containerd/cmd/ctr-cli/pack" "github.com/naoki9911/fuse-diff-containerd/cmd/ctr-cli/patch" "github.com/naoki9911/fuse-diff-containerd/cmd/ctr-cli/pull" + "github.com/naoki9911/fuse-diff-containerd/cmd/ctr-cli/show" "github.com/urfave/cli/v2" ) @@ -38,7 +39,6 @@ func NewApp() *cli.App { pack.Command(), load.Command(), pull.Command(), - merge.Command(), } return app @@ -52,6 +52,8 @@ func dimgCommand() *cli.Command { Subcommands: []*cli.Command{ patch.DimgCommand(), diff.DimgCommand(), + merge.DimgCommand(), + show.DimgCommand(), }, } return &cmd @@ -65,6 +67,8 @@ func cdimgCommand() *cli.Command { Subcommands: []*cli.Command{ patch.CdimgCommand(), diff.CdimgCommand(), + merge.CdimgCommand(), + show.CdimgCommand(), }, } return &cmd diff --git a/cmd/ctr-cli/merge/merge.go b/cmd/ctr-cli/merge/merge.go index 5623f16..fd3658e 100644 --- a/cmd/ctr-cli/merge/merge.go +++ b/cmd/ctr-cli/merge/merge.go @@ -14,33 +14,42 @@ import ( var logger = log.G(context.TODO()) -var ( - Flags = []cli.Flag{ - &cli.StringFlag{ - Name: "lowerDimg", - Usage: "path to lower dimg", - Required: true, - }, - &cli.StringFlag{ - Name: "upperDimg", - Usage: "path to upper dimg", - Required: true, - }, - &cli.StringFlag{ - Name: "outDimg", - Usage: "path to merged dimg", - Required: true, +func DimgCommand() *cli.Command { + cmd := cli.Command{ + Name: "merge", + Usage: "merge dimgs", + Action: func(context *cli.Context) error { + return dimgAction(context) }, - &cli.BoolFlag{ - Name: "benchmark", - Usage: "enable benchmark", - Value: false, - Required: false, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "lowerDimg", + Usage: "path to lower dimg", + Required: true, + }, + &cli.StringFlag{ + Name: "upperDimg", + Usage: "path to upper dimg", + Required: true, + }, + &cli.StringFlag{ + Name: "outDimg", + Usage: "path to merged dimg", + Required: true, + }, + &cli.BoolFlag{ + Name: "benchmark", + Usage: "enable benchmark", + Value: false, + Required: false, + }, }, } -) -func Action(c *cli.Context) error { + return &cmd +} + +func dimgAction(c *cli.Context) error { logger.Logger.SetLevel(logrus.WarnLevel) lowerDimg := c.String("lowerDimg") upperDimg := c.String("upperDimg") @@ -69,7 +78,7 @@ func Action(c *cli.Context) error { panic(err) } defer mergeFile.Close() - err = image.MergeDimg(lowerDimg, upperDimg, mergeFile) + _, err = image.MergeDimg(lowerDimg, upperDimg, mergeFile) if err != nil { panic(err) } @@ -93,15 +102,90 @@ func Action(c *cli.Context) error { return nil } -func Command() *cli.Command { +func CdimgCommand() *cli.Command { cmd := cli.Command{ Name: "merge", - Usage: "merge dimgs", + Usage: "merge cdimgs", Action: func(context *cli.Context) error { - return Action(context) + return cdimgAction(context) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "lowerCdimg", + Usage: "path to lower cdimg", + Required: true, + }, + &cli.StringFlag{ + Name: "upperCdimg", + Usage: "path to upper cdimg", + Required: true, + }, + &cli.StringFlag{ + Name: "outCdimg", + Usage: "path to merged cdimg", + Required: true, + }, + &cli.BoolFlag{ + Name: "benchmark", + Usage: "enable benchmark", + Value: false, + Required: false, + }, }, - Flags: Flags, } return &cmd } + +func cdimgAction(c *cli.Context) error { + logger.Logger.SetLevel(logrus.WarnLevel) + lowerCdimg := c.String("lowerCdimg") + upperCdimg := c.String("upperCdimg") + outCdimg := c.String("outCdimg") + enableBench := c.Bool("benchmark") + logger.WithFields(logrus.Fields{ + "lowerCdimg": lowerCdimg, + "upperCdimg": upperCdimg, + "outCdimg": outCdimg, + }).Info("starting to merge") + + var b *benchmark.Benchmark = nil + var err error + if enableBench { + b, err = benchmark.NewBenchmark("./benchmark.log") + if err != nil { + return err + } + defer b.Close() + } + + start := time.Now() + + mergeFile, err := os.Create(outCdimg) + if err != nil { + panic(err) + } + defer mergeFile.Close() + err = image.MergeCdimg(lowerCdimg, upperCdimg, mergeFile) + if err != nil { + panic(err) + } + + if b != nil { + metric := benchmark.Metric{ + TaskName: "patch", + ElapsedMilli: int(time.Since(start).Milliseconds()), + Labels: []string{ + "lowerCdimg:" + lowerCdimg, + "upperCdimg:" + upperCdimg, + "outCdimg:" + outCdimg, + }, + } + err = b.AppendResult(metric) + if err != nil { + panic(err) + } + } + logger.Info("merge done") + return nil +} diff --git a/cmd/ctr-cli/patch/patch.go b/cmd/ctr-cli/patch/patch.go index 3962214..5ccfbce 100644 --- a/cmd/ctr-cli/patch/patch.go +++ b/cmd/ctr-cli/patch/patch.go @@ -81,7 +81,7 @@ func dimgAction(c *cli.Context) error { dimgHeader := dimgFile.Header() start := time.Now() - err = image.ApplyPatch(baseDir, outDir, &dimgHeader.FileEntry, dimgFile, dimgHeader.BaseId == "") + err = image.ApplyPatch(baseDir, outDir, &dimgHeader.FileEntry, dimgFile, dimgHeader.ParentId == "") if err != nil { panic(err) } @@ -172,7 +172,7 @@ func cdimgAction(c *cli.Context) error { dimgHeader := dimgFile.Header() start := time.Now() - err = image.ApplyPatch(baseDir, outDir, &dimgHeader.FileEntry, dimgFile, dimgHeader.BaseId == "") + err = image.ApplyPatch(baseDir, outDir, &dimgHeader.FileEntry, dimgFile, dimgHeader.ParentId == "") if err != nil { panic(err) } diff --git a/cmd/ctr-cli/show/show.go b/cmd/ctr-cli/show/show.go new file mode 100644 index 0000000..624b1b3 --- /dev/null +++ b/cmd/ctr-cli/show/show.go @@ -0,0 +1,87 @@ +package show + +import ( + "context" + "fmt" + + "github.com/containerd/containerd/log" + "github.com/naoki9911/fuse-diff-containerd/pkg/image" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +var logger = log.G(context.TODO()) + +func DimgCommand() *cli.Command { + cmd := cli.Command{ + Name: "show", + Usage: "show dimg info", + Action: func(context *cli.Context) error { + return dimgAction(context) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dimg", + Usage: "path to dimg", + Required: true, + }, + }, + } + + return &cmd +} + +func dimgAction(c *cli.Context) error { + logger.Logger.SetLevel(logrus.WarnLevel) + dimgPath := c.String("dimg") + + dimgFile, err := image.OpenDimgFile(dimgPath) + if err != nil { + return err + } + defer dimgFile.Close() + header := dimgFile.Header() + fmt.Printf("ID: %s\n", header.Id) + fmt.Printf("ParentID: %s\n", header.ParentId) + return nil +} + +func CdimgCommand() *cli.Command { + cmd := cli.Command{ + Name: "show", + Usage: "show cdimg info", + Action: func(context *cli.Context) error { + return cdimgAction(context) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cdimg", + Usage: "path to cdimg", + Required: true, + }, + }, + } + + return &cmd +} + +func cdimgAction(c *cli.Context) error { + logger.Logger.SetLevel(logrus.WarnLevel) + cdimgPath := c.String("cdimg") + cdimgFile, err := image.OpenCdimgFile(cdimgPath) + if err != nil { + return err + } + defer cdimgFile.Close() + header := cdimgFile.Header + head := header.Head + + fmt.Printf("Manifest Digest: %s\n", head.ManifestDigest) + fmt.Printf("Manifest: %v\n", header.Manifest) + fmt.Printf("Config: %v\n", header.Config) + fmt.Printf("DimgDigest: %s\n", header.DimgDigest) + fmt.Printf("DimgID: %s\n", cdimgFile.Dimg.Header().Id) + fmt.Printf("DimgParentID: %s\n", cdimgFile.Dimg.Header().ParentId) + + return nil +} diff --git a/cmd/fuse-diff/main.go b/cmd/fuse-diff/main.go index 712e30d..4afdb34 100644 --- a/cmd/fuse-diff/main.go +++ b/cmd/fuse-diff/main.go @@ -13,7 +13,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "runtime/pprof" "syscall" "time" @@ -61,8 +60,10 @@ func main() { bench := flag.Bool("benchmark", false, "measure benchmark") cpuprofile := flag.String("cpuprofile", "", "write cpu profile to this file") memprofile := flag.String("memprofile", "", "write memory profile to this file") - baseDimg := flag.String("baseDimg", "", "base directory to be patched") - diffDimg := flag.String("diffDimg", "", "patch directory") + parentDimg := flag.String("parentDimg", "", "path to parent dimg") + diffDimg := flag.String("diffDimg", "", "path to diff dimg") + parentCdimg := flag.String("parentCdimg", "", "path to parent cdimg") + diffCdimg := flag.String("diffCdimg", "", "path to diff cdimg") flag.Parse() if flag.NArg() < 1 { fmt.Printf("usage: %s MOUNTPOINT\n", path.Base(os.Args[0])) @@ -102,45 +103,48 @@ func main() { fmt.Printf("Note: You must unmount gracefully, otherwise the profile file(s) will stay empty!\n") } - if *diffDimg == "" { - fmt.Println("please specify diffDimg") + if *diffDimg == "" && *diffCdimg == "" { + fmt.Println("please specify '--diffDimg' or '--diffCdimg'") os.Exit(1) } - diffDimgAbs, err := filepath.Abs(*diffDimg) - if err != nil { - panic(err) - } - - diffImageFile, err := image.OpenDimgFile(diffDimgAbs) - if err != nil { - panic(err) - } - var baseImageFile *image.DimgFile = nil - baseNeeded := diffImageFile.Header().BaseId != "" - if baseNeeded { - var baseDimgAbs string - if *baseDimg != "" { - baseDimgAbs, err = filepath.Abs(*baseDimg) - if err != nil { - panic(err) - } - } else { - imageStore, _ := filepath.Split(diffDimgAbs) - baseDimgAbs = filepath.Join(imageStore, diffImageFile.Header().BaseId+".dimg") + var diffImageFile *image.DimgFile = nil + if *diffDimg != "" { + diffImageFile, err = image.OpenDimgFile(*diffDimg) + if err != nil { + panic(err) } - baseImageFile, err = image.OpenDimgFile(baseDimgAbs) + defer diffImageFile.Close() + } else { + diffCdimgFile, err := image.OpenCdimgFile(*diffCdimg) if err != nil { panic(err) } - baseNeeded = false + defer diffCdimgFile.Close() + diffImageFile = diffCdimgFile.Dimg } - if baseNeeded && *baseDimg == "" { - fmt.Println("please specify baseDimg") + parentNeeded := diffImageFile.Header().ParentId != "" + if parentNeeded && *parentDimg == "" && *parentCdimg == "" { + fmt.Println("please specify '--parentDimg' or '--parentCdimg'") os.Exit(1) } + var parentImageFile *image.DimgFile = nil + if *parentDimg != "" { + parentImageFile, err = image.OpenDimgFile(*parentDimg) + if err != nil { + panic(err) + } + defer parentImageFile.Close() + } else { + parentCdimgFile, err := image.OpenCdimgFile(*parentCdimg) + if err != nil { + panic(err) + } + parentImageFile = parentCdimgFile.Dimg + } + sec := time.Second opts := &fs.Options{ // These options are to be compatible with libfuse defaults, @@ -166,7 +170,7 @@ func main() { // Leave file permissions on "000" files as-is opts.NullPermissions = true - di3fsRoot, err := di3fs.NewDi3fsRoot(opts, []*image.DimgFile{baseImageFile}, diffImageFile) + di3fsRoot, err := di3fs.NewDi3fsRoot(opts, []*image.DimgFile{parentImageFile}, diffImageFile) if err != nil { log.Fatalf("creating Di3fsRoot failed: %v\n", err) } @@ -183,7 +187,7 @@ func main() { TaskName: "di3fs", ElapsedMilli: int(elapsedMilli), Labels: []string{ - "base:" + *baseDimg, + "parent:" + *parentDimg, "patch:" + *diffDimg, }, } diff --git a/cmd/server/main.go b/cmd/server/main.go index 2494b2d..1a67bad 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -184,6 +184,7 @@ func handleGetUpdateData(w http.ResponseWriter, r *http.Request) { logger.WithField("Diffs", pathStr).Info("Diff Data to be transfered found") diffDataBytes := bytes.Buffer{} + diffHeader := &image.DimgHeader{} configPath := "" if len(path) == 2 { @@ -195,6 +196,19 @@ func handleGetUpdateData(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) return } + diffHeader, _, err = image.LoadDimgHeader(diffF) + if err != nil { + logger.Errorf("failed to lead DimgHeader: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = diffF.Seek(0, 0) + if err != nil { + logger.Errorf("failed to seek dimg File: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = io.Copy(&diffDataBytes, diffF) if err != nil { logger.Errorf("failed to load=%v", err) @@ -213,7 +227,7 @@ func handleGetUpdateData(w http.ResponseWriter, r *http.Request) { upperDiff := DiffDatas[upperTag] logger.WithFields(logrus.Fields{"lower": lowerFileName, "upper": upperDiff.FileName}).Info("merge") diffDataBytes = bytes.Buffer{} - err = image.MergeDimg(lowerFileName, upperDiff.FileName, &diffDataBytes) + diffHeader, err = image.MergeDimg(lowerFileName, upperDiff.FileName, &diffDataBytes) if err != nil { logger.Errorf("failed to merge=%v", err) w.WriteHeader(http.StatusInternalServerError) @@ -275,9 +289,16 @@ func handleGetUpdateData(w http.ResponseWriter, r *http.Request) { return } defer configFile.Close() - err = image.PackIo(configFile, diffDataBytes.Bytes(), w) + + err = image.WriteCdimgHeader(configFile, diffHeader.Id, int64(diffDataBytes.Len()), w) + if err != nil { + logger.Errorf("failed to cdimg header: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = io.Copy(w, &diffDataBytes) if err != nil { - logger.Errorf("failed to pack config file err=%v", err) + logger.Errorf("failed to write dimg: %v", err) w.WriteHeader(http.StatusInternalServerError) return } diff --git a/pkg/di3fs/di3fs.go b/pkg/di3fs/di3fs.go index 4bed244..0e0e802 100644 --- a/pkg/di3fs/di3fs.go +++ b/pkg/di3fs/di3fs.go @@ -285,6 +285,9 @@ func newNode(fe *image.FileEntry, baseFE []*image.FileEntry, root *Di3fsRoot) *D func NewDi3fsRoot(opts *fs.Options, baseImages []*image.DimgFile, diffImage *image.DimgFile) (Di3fsRoot, error) { baseFEs := make([]*image.FileEntry, 0) for i := range baseImages { + if baseImages[i] == nil { + continue + } baseFEs = append(baseFEs, &baseImages[i].Header().FileEntry) } rootNode := newNode(&diffImage.Header().FileEntry, baseFEs, nil) @@ -312,24 +315,24 @@ func Do(diffImagePath, mountPath string) error { panic(err) } - baseImageFiles := make([]*image.DimgFile, 0) + parentImageFiles := make([]*image.DimgFile, 0) diffImageFile, err := image.OpenDimgFile(diffImagePathAbs) if err != nil { panic(err) } - baseImageId := diffImageFile.Header().BaseId - for baseImageId != "" { + parentImageId := diffImageFile.Header().ParentId + for parentImageId != "" { imageStore, _ := filepath.Split(diffImagePathAbs) - baseImagePath := filepath.Join(imageStore, baseImageId+".dimg") - baseImageFile, err := image.OpenDimgFile(baseImagePath) + parentImagePath := filepath.Join(imageStore, parentImageId.String()+".dimg") + parentImageFile, err := image.OpenDimgFile(parentImagePath) if err != nil { panic(err) } - baseImageFiles = append(baseImageFiles, baseImageFile) - baseImageId = baseImageFile.Header().BaseId - log.Infof("baseImage %s is loaded", baseImageId) + parentImageFiles = append(parentImageFiles, parentImageFile) + parentImageId = parentImageFile.Header().ParentId + log.Infof("parentImage %s is loaded", parentImageId) } sec := time.Second @@ -343,7 +346,7 @@ func Do(diffImagePath, mountPath string) error { opts.MountOptions.Name = "fuse-diff" opts.NullPermissions = true - di3fsRoot, err := NewDi3fsRoot(opts, baseImageFiles, diffImageFile) + di3fsRoot, err := NewDi3fsRoot(opts, parentImageFiles, diffImageFile) if err != nil { log.Fatalf("creating Di3fsRoot failed: %v\n", err) } diff --git a/pkg/image/cdimg.go b/pkg/image/cdimg.go index f31e144..0e25452 100644 --- a/pkg/image/cdimg.go +++ b/pkg/image/cdimg.go @@ -138,12 +138,27 @@ func PackCdimg(configPath, dimgPath, outPath string) error { defer dimgFile.Close() logger.Debugf("opened dimgFile %q", dimgPath) - dimgBytes, err := io.ReadAll(dimgFile) + dimgStat, err := dimgFile.Stat() if err != nil { return err } - err = PackIo(configFile, dimgBytes, outFile) + dimgHeader, _, err := LoadDimgHeader(dimgFile) + if err != nil { + return err + } + + err = WriteCdimgHeader(configFile, dimgHeader.Id, dimgStat.Size(), outFile) + if err != nil { + return err + } + + _, err = dimgFile.Seek(0, 0) + if err != nil { + return err + } + + _, err = io.Copy(outFile, dimgFile) if err != nil { return err } @@ -151,19 +166,15 @@ func PackCdimg(configPath, dimgPath, outPath string) error { return nil } -func PackIo(configReader io.Reader, dimg []byte, out io.Writer) error { +func WriteCdimgHeader(configReader io.Reader, dimgId digest.Digest, dimgSize int64, out io.Writer) error { head := CdimgHeadHeader{} outBytes := bytes.Buffer{} config, err := loadConfigFromReader(configReader) if err != nil { return err } - dimgFileSize, dimgFilDigest, err := utils.GetSizeAndDigest(dimg) - if err != nil { - return err - } - config.RootFS.DiffIDs = []digest.Digest{*dimgFilDigest} + config.RootFS.DiffIDs = []digest.Digest{dimgId} configBytes, err := json.Marshal(config) if err != nil { return err @@ -184,8 +195,8 @@ func PackIo(configReader io.Reader, dimg []byte, out io.Writer) error { Layers: []v1.Descriptor{ { MediaType: v1.MediaTypeImageLayer, - Size: dimgFileSize, - Digest: *dimgFilDigest, + Size: dimgSize, + Digest: dimgId, }, }, } @@ -212,7 +223,7 @@ func PackIo(configReader io.Reader, dimg []byte, out io.Writer) error { } logger.Debugf("compressed config (size=%d)", head.ConfigSize) - head.DimgSize = dimgFileSize + head.DimgSize = dimgSize headCompressedBytes, err := head.pack() if err != nil { @@ -231,11 +242,6 @@ func PackIo(configReader io.Reader, dimg []byte, out io.Writer) error { if err != nil { return err } - _, err = out.Write(dimg) - if err != nil { - return err - } - logger.Debugf("written contents") return nil } diff --git a/pkg/image/diff.go b/pkg/image/diff.go index 5b68cd1..fc0bdfe 100644 --- a/pkg/image/diff.go +++ b/pkg/image/diff.go @@ -2,7 +2,6 @@ package image import ( "bytes" - "crypto/sha256" "fmt" "io" "os" @@ -45,20 +44,9 @@ func GenerateDiffFromDimg(oldDimgPath, newDimgPath, diffDimgPath string, isBinar return err } - h := sha256.New() - baseImg, err := os.Open(oldDimgPath) - if err != nil { - panic(err) - } - defer baseImg.Close() - _, err = io.Copy(h, baseImg) - if err != nil { - panic(err) - } - baseId := fmt.Sprintf("sha256:%x", h.Sum(nil)) - header := DimgHeader{ - BaseId: baseId, + Id: newDimg.Header().Id, + ParentId: oldDimg.Header().Id, FileEntry: newDimg.header.FileEntry, } @@ -97,16 +85,10 @@ func GenerateDiffFromCdimg(oldCdimgPath, newCdimgPath, diffCdimgPath string, isB return err } - h := sha256.New() - err = oldCdimg.WriteDimg(h) - if err != nil { - panic(err) - } - baseId := fmt.Sprintf("sha256:%x", h.Sum(nil)) - diffDimgOut := bytes.Buffer{} header := DimgHeader{ - BaseId: baseId, + Id: newDimg.Header().Id, + ParentId: oldDimg.Header().Id, FileEntry: newDimg.header.FileEntry, } err = WriteDimg(&diffDimgOut, &header, &diffOut) @@ -114,9 +96,13 @@ func GenerateDiffFromCdimg(oldCdimgPath, newCdimgPath, diffCdimgPath string, isB return fmt.Errorf("failed to write dimg: %v", err) } - err = PackIo(bytes.NewBuffer(newCdimg.Header.ConfigBytes), diffDimgOut.Bytes(), diffCdimg) + err = WriteCdimgHeader(bytes.NewBuffer(newCdimg.Header.ConfigBytes), header.Id, int64(diffDimgOut.Len()), diffCdimg) if err != nil { - return fmt.Errorf("failed to pack cdimg: %v", err) + return fmt.Errorf("failed to cdimg header: %v", err) + } + _, err = io.Copy(diffCdimg, &diffDimgOut) + if err != nil { + return fmt.Errorf("failed to write dimg: %v", err) } return nil diff --git a/pkg/image/dimg.go b/pkg/image/dimg.go index 61bca6c..cb1cb61 100644 --- a/pkg/image/dimg.go +++ b/pkg/image/dimg.go @@ -9,11 +9,13 @@ import ( "os" "github.com/klauspost/compress/zstd" + "github.com/opencontainers/go-digest" ) type DimgHeader struct { - BaseId string `json:"baseID"` - FileEntry FileEntry `json:"fileEntry"` + Id digest.Digest `json:"id"` // generated when the full image generated + ParentId digest.Digest `json:"parentID"` + FileEntry FileEntry `json:"fileEntry"` } type DimgFile struct { diff --git a/pkg/image/merge.go b/pkg/image/merge.go index 1ae9c53..add125d 100644 --- a/pkg/image/merge.go +++ b/pkg/image/merge.go @@ -771,7 +771,7 @@ func MergeDiffDimg(lowerEntry, upperEntry *FileEntry, lowerDiff, upperDiff strin return nil } -func MergeDimg(lowerDimg, upperDimg string, merged io.Writer) error { +func MergeDimg(lowerDimg, upperDimg string, merged io.Writer) (*DimgHeader, error) { lowerImgFile, err := OpenDimgFile(lowerDimg) if err != nil { panic(err) @@ -792,13 +792,61 @@ func MergeDimg(lowerDimg, upperDimg string, merged io.Writer) error { } header := DimgHeader{ - BaseId: lowerImgFile.Header().BaseId, + Id: upperImgFile.Header().Id, + ParentId: lowerImgFile.Header().ParentId, FileEntry: *mergedEntry, } err = WriteDimg(merged, &header, &tmp) + if err != nil { + return nil, fmt.Errorf("failed to write to dimg: %v", err) + } + return &header, nil +} + +func MergeCdimg(lowerCdimg, upperCdimg string, merged io.Writer) error { + lowerCdimgFile, err := OpenCdimgFile(lowerCdimg) + if err != nil { + panic(err) + } + defer lowerCdimgFile.Close() + lowerDimg := lowerCdimgFile.Dimg + + upperCdimgFile, err := OpenCdimgFile(upperCdimg) + if err != nil { + panic(err) + } + defer upperCdimgFile.Close() + upperDimg := upperCdimgFile.Dimg + + tmp := bytes.Buffer{} + mergedEntry := upperDimg.Header().FileEntry.DeepCopy() + lowerFE := &lowerDimg.Header().FileEntry + upperFE := &upperDimg.Header().FileEntry + err = MergeDiffDimg(lowerFE, upperFE, lowerFE.Name, upperFE.Name, lowerDimg, upperDimg, mergedEntry, &tmp) + if err != nil { + panic(err) + } + + header := DimgHeader{ + Id: upperDimg.Header().Id, + ParentId: lowerDimg.Header().ParentId, + FileEntry: *mergedEntry, + } + + mergedDimg := bytes.Buffer{} + err = WriteDimg(&mergedDimg, &header, &tmp) if err != nil { return fmt.Errorf("failed to write to dimg: %v", err) } + + err = WriteCdimgHeader(bytes.NewBuffer(upperCdimgFile.Header.ConfigBytes), header.Id, int64(mergedDimg.Len()), merged) + if err != nil { + return fmt.Errorf("failed to cdimg header: %v", err) + } + _, err = io.Copy(merged, &mergedDimg) + if err != nil { + return fmt.Errorf("failed to write dimg: %v", err) + } return nil } diff --git a/pkg/image/pack.go b/pkg/image/pack.go index 6c82a7d..31ad103 100644 --- a/pkg/image/pack.go +++ b/pkg/image/pack.go @@ -7,6 +7,8 @@ import ( "io/fs" "os" "path" + + "github.com/opencontainers/go-digest" ) func compressFileWithZstd(path string) ([]byte, error) { @@ -172,8 +174,11 @@ func PackDir(dirPath, outDimgPath string) error { return err } + bodyDigest := digest.FromBytes(outBody.Bytes()) + header := DimgHeader{ - BaseId: "", + Id: bodyDigest, + ParentId: digest.Digest(""), FileEntry: *entry, } diff --git a/tests/.gitignore b/tests/.gitignore index a34b883..30ae57f 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,2 @@ /benchmark_* +/benchmark diff --git a/tests/bench_impl.sh b/tests/bench_impl.sh index 19f96fb..b87b93b 100755 --- a/tests/bench_impl.sh +++ b/tests/bench_impl.sh @@ -66,7 +66,7 @@ for ((i=0; i < $(expr ${#IMAGE_VERSIONS[@]} - 1); i++));do for ((j=0; j < $RUN_NUM; j++));do NOW_COUNT=$(expr $j + 1) echo "Benchmark di3fs $DIFF_NAME binary-diff ($NOW_COUNT/$RUN_NUM)" - $BIN_FUSE --baseDimg=./$LOWER.dimg --diffDimg=./diff_$DIFF_NAME.dimg --benchmark=true /tmp/fuse >/dev/null 2>&1 & + $BIN_FUSE --parentDimg=./$LOWER.dimg --diffDimg=./diff_$DIFF_NAME.dimg --benchmark=true /tmp/fuse >/dev/null 2>&1 & sleep 1 if [ $j -eq 0 ]; then diff -r $UPPER /tmp/fuse --no-dereference @@ -93,14 +93,14 @@ MERGED=$IMAGE_LOWER-$IMAGE_UPPER for ((j=0; j < $RUN_NUM; j++));do NOW_COUNT=$(expr $j + 1) echo "Benchmark merge $MERGE_LOWER and $MERGE_UPPER to $MERGED ($NOW_COUNT/$RUN_NUM)" - $BIN_CTR_CLI merge --lowerDimg=./diff_$MERGE_LOWER.dimg --upperDimg=./diff_$MERGE_UPPER.dimg --outDimg=./diff_merged_$MERGED.dimg --benchmark + $BIN_CTR_CLI dimg merge --lowerDimg=./diff_$MERGE_LOWER.dimg --upperDimg=./diff_$MERGE_UPPER.dimg --outDimg=./diff_merged_$MERGED.dimg --benchmark done echo "Testing merged $MERGED" $BIN_CTR_CLI dimg patch --baseDir=./$IMAGE_LOWER --outDir=./$IMAGE_UPPER-merged --diffDimg=./diff_merged_$MERGED.dimg diff -r $IMAGE_UPPER $IMAGE_UPPER-merged --no-dereference ls -l diff_merged_$MERGED.dimg -$BIN_FUSE --baseDimg=./$IMAGE_LOWER.dimg --diffDimg=./diff_merged_$MERGED.dimg /tmp/fuse >/dev/null 2>&1 & +$BIN_FUSE --parentDimg=./$IMAGE_LOWER.dimg --diffDimg=./diff_merged_$MERGED.dimg /tmp/fuse >/dev/null 2>&1 & sleep 1 diff -r $IMAGE_UPPER /tmp/fuse --no-dereference fusermount3 -u /tmp/fuse diff --git a/tests/bench_patch_impl.sh b/tests/bench_patch_impl.sh index 85fb506..7bd8034 100755 --- a/tests/bench_patch_impl.sh +++ b/tests/bench_patch_impl.sh @@ -45,7 +45,7 @@ for ((i=0; i < $(expr ${#IMAGE_VERSIONS[@]} - 1); i++));do for ((j=0; j < $RUN_NUM; j++));do NOW_COUNT=$(expr $j + 1) echo "Benchmark di3fs $DIFF_NAME binary-diff ($NOW_COUNT/$RUN_NUM)" - $BIN_FUSE --baseDimg=./$LOWER.dimg --diffDimg=./diff_$DIFF_NAME.dimg --benchmark=true /tmp/fuse >/dev/null 2>&1 & + $BIN_FUSE --parentDimg=./$LOWER.dimg --diffDimg=./diff_$DIFF_NAME.dimg --benchmark=true /tmp/fuse >/dev/null 2>&1 & sleep 5 if [ $j -eq 0 ]; then diff -r $UPPER /tmp/fuse --no-dereference