diff --git a/cmd/server/main.go b/cmd/server/main.go index 7843f2a..78047c2 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -110,7 +110,7 @@ func (ds *diffServer) handlePostDiffData(w http.ResponseWriter, r *http.Request) } baseVersion = baseVer.String() } - DiffDataGraphs[diffData.ImageName].Add(baseVersion, v.String(), 1) + DiffDataGraphs[diffData.ImageName].Add(baseVersion, v.String(), fmt.Sprintf("%s-%s", baseVersion, v.String()), 1) DiffDatas[getDiffTag(diffData.ImageName, baseVersion, v.String())] = diffData logger.WithFields(logrus.Fields{"baseVersion": baseVersion, "version": v.String(), "Name": diffData.ImageName}).Infof("added diffData to dependency graph") @@ -179,7 +179,7 @@ func (ds *diffServer) handleGetUpdateData(w http.ResponseWriter, r *http.Request } logger.WithFields(logrus.Fields{"baseVersion": baseVersion, "version": targetVersion.String(), "Name": req.RequestImage.Name}).Infof("start to find best diffs") - path, err := graph.ShortestPath(baseVersion, targetVersion.String()) + path, _, err := graph.ShortestPath(baseVersion, targetVersion.String()) if err != nil { logger.Errorf("DiffDatas commbination for %s not found (base=%s target=%s)", req.RequestImage.Name, baseVersion, targetVersion.String()) w.WriteHeader(http.StatusNotFound) diff --git a/cmd/snapshotter/fs.go b/cmd/snapshotter/fs.go index abd2fc2..773071f 100644 --- a/cmd/snapshotter/fs.go +++ b/cmd/snapshotter/fs.go @@ -69,10 +69,11 @@ func (f *Di3FSManager) Mount(ctx context.Context, mountpoint string, labels map[ return errdefs.ErrNotFound } - dimgPaths, err := f.dimgStore.GetDimgPaths(digest.Digest(d)) + dimgPaths, err := f.dimgStore.GetDimgPathsWithDimgDigest(digest.Digest(d)) if err != nil { return fmt.Errorf("failed to get dimg paths for %s: %v", d, err) } + log.G(ctx).Infof("dimg paths: %v", dimgPaths) err = f.mountDImg(ctx, mountpoint, dimgPaths) if err != nil { diff --git a/pkg/algorithm/dijkstra.go b/pkg/algorithm/dijkstra.go index 46db846..c937202 100644 --- a/pkg/algorithm/dijkstra.go +++ b/pkg/algorithm/dijkstra.go @@ -15,10 +15,11 @@ type Node struct { done bool // 処理済みかを表すフラグ cost int // このノードにたどり着くのに必要だったコスト prev *Node // このノードにたどりつくのに使われたノード + via *Edge // the edge used to reach this node } func NewNode(name string) *Node { - node := &Node{name, []*Edge{}, false, -1, nil} + node := &Node{name, []*Edge{}, false, -1, nil, nil} return node } @@ -33,15 +34,20 @@ func (n *Node) GetName() string { // エッジ type Edge struct { + name string next *Node // 次に移動できるノード cost int // 移動にかかるコスト } -func NewEdge(next *Node, cost int) *Edge { - edge := &Edge{next, cost} +func NewEdge(name string, next *Node, cost int) *Edge { + edge := &Edge{name, next, cost} return edge } +func (e *Edge) GetName() string { + return e.name +} + // 有向グラフ type DirectedGraph struct { nodes map[string]*Node @@ -58,8 +64,8 @@ func (self *DirectedGraph) Print() { } } -// グラフの要素を追加する (接続元ノード名、接続先ノード名、移動にかかるコスト) -func (self *DirectedGraph) Add(src, dst string, cost int) { +// グラフの要素を追加する (接続元ノード名、接続先ノード名、Edge Name, 移動にかかるコスト) +func (self *DirectedGraph) Add(src, dst, edgeName string, cost int) { // ノードが既にある場合は追加しない srcNode, ok := self.nodes[src] if !ok { @@ -74,12 +80,12 @@ func (self *DirectedGraph) Add(src, dst string, cost int) { } // ノードをエッジでつなぐ - edge := NewEdge(dstNode, cost) + edge := NewEdge(edgeName, dstNode, cost) srcNode.AddEdge(edge) } // スタートとゴールを指定して最短経路を求める -func (self *DirectedGraph) ShortestPath(start string, goal string) (ret []*Node, err error) { +func (self *DirectedGraph) ShortestPath(start string, goal string) (ret []*Node, via []*Edge, err error) { // 名前からスタート地点のノードを取得する startNode := self.nodes[start] @@ -92,7 +98,7 @@ func (self *DirectedGraph) ShortestPath(start string, goal string) (ret []*Node, // 次に処理するノードが見つからなければ終了 if err != nil { - return nil, errors.New("Goal not found") + return nil, nil, errors.New("Goal not found") } // ゴールまで到達した @@ -107,11 +113,13 @@ func (self *DirectedGraph) ShortestPath(start string, goal string) (ret []*Node, // ゴールから逆順にスタートまでノードをたどっていく n := self.nodes[goal] ret_rev := make([]*Node, 0) + viaEdgesRev := make([]*Edge, 0) for { ret_rev = append(ret_rev, n) if n.name == start { break } + viaEdgesRev = append(viaEdgesRev, n.via) n = n.prev } @@ -119,20 +127,25 @@ func (self *DirectedGraph) ShortestPath(start string, goal string) (ret []*Node, ret = append(ret, ret_rev[len(ret_rev)-i-1]) } + for i := range viaEdgesRev { + via = append(via, viaEdgesRev[len(viaEdgesRev)-i-1]) + } + // Reset all nodes for i := range self.nodes { self.nodes[i].done = false self.nodes[i].cost = -1 self.nodes[i].prev = nil + self.nodes[i].via = nil } - return ret, nil + return ret, via, nil } // つながっているノードのコストを計算する func (self *DirectedGraph) calc(node *Node) { // ノードにつながっているエッジを取得する - for _, edge := range node.edges { + for i, edge := range node.edges { nextNode := edge.next // 既に処理済みのノードならスキップする @@ -146,6 +159,7 @@ func (self *DirectedGraph) calc(node *Node) { // 既に見つかっている経路よりもコストが小さければ処理中のノードを遷移元として記録する nextNode.cost = cost nextNode.prev = node + nextNode.via = node.edges[i] } } diff --git a/pkg/algorithm/dijkstra_test.go b/pkg/algorithm/dijkstra_test.go index 5ae4e9f..8af9ba6 100644 --- a/pkg/algorithm/dijkstra_test.go +++ b/pkg/algorithm/dijkstra_test.go @@ -10,15 +10,20 @@ import ( func TestDijkstra(t *testing.T) { g := algorithm.NewDirectedGraph() - g.Add("1.23.1", "1.23.2", 1) - g.Add("1.23.2", "1.23.3", 1) - g.Add("1.23.3", "1.23.4", 1) - g.Add("1.23.1", "1.23.3", 1) + g.Add("1.23.1", "1.23.2", "hoge1", 1) + g.Add("1.23.2", "1.23.3", "hoge2", 1) + g.Add("1.23.3", "1.23.4", "hoge3", 1) + g.Add("1.23.1", "1.23.3", "hoge4", 1) - path, err := g.ShortestPath("1.23.1", "1.23.4") + path, via, err := g.ShortestPath("1.23.1", "1.23.4") assert.Equal(t, nil, err) assert.Equal(t, 3, len(path)) assert.Equal(t, "1.23.1", path[0].GetName()) assert.Equal(t, "1.23.3", path[1].GetName()) assert.Equal(t, "1.23.4", path[2].GetName()) + + assert.Equal(t, nil, err) + assert.Equal(t, 2, len(via)) + assert.Equal(t, "hoge4", via[0].GetName()) + assert.Equal(t, "hoge3", via[1].GetName()) } diff --git a/pkg/image/store.go b/pkg/image/store.go index 86764dc..77f5f0c 100644 --- a/pkg/image/store.go +++ b/pkg/image/store.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "path/filepath" + "sync" + "github.com/naoki9911/fuse-diff-containerd/pkg/algorithm" "github.com/opencontainers/go-digest" ) @@ -15,7 +17,8 @@ type DimgEntry struct { type DimgStore struct { storeDir string - dimgIds map[digest.Digest]*DimgEntry + storeLock sync.Mutex + dimgGraph *algorithm.DirectedGraph dimgDigests map[digest.Digest]*DimgEntry } @@ -27,7 +30,8 @@ func NewDimgStore(storeDir string) (*DimgStore, error) { s := &DimgStore{ storeDir: storeDir, - dimgIds: map[digest.Digest]*DimgEntry{}, + storeLock: sync.Mutex{}, + dimgGraph: algorithm.NewDirectedGraph(), dimgDigests: map[digest.Digest]*DimgEntry{}, } @@ -41,12 +45,15 @@ func NewDimgStore(storeDir string) (*DimgStore, error) { } func (ds *DimgStore) Walk() error { + ds.storeLock.Lock() + defer ds.storeLock.Unlock() + dirs, err := os.ReadDir(ds.storeDir) if err != nil { return fmt.Errorf("failed to ReadDir %s: %v", ds.storeDir, err) } - ds.dimgIds = map[digest.Digest]*DimgEntry{} + ds.dimgGraph = algorithm.NewDirectedGraph() ds.dimgDigests = map[digest.Digest]*DimgEntry{} for _, dir := range dirs { @@ -67,7 +74,7 @@ func (ds *DimgStore) Walk() error { DimgHeader: *header, Path: fPath, } - ds.dimgIds[header.Id] = entry + ds.dimgGraph.Add(string(header.ParentId), header.Id.String(), string(header.Digest()), 1) ds.dimgDigests[header.Digest()] = entry } @@ -75,6 +82,9 @@ func (ds *DimgStore) Walk() error { } func (ds *DimgStore) AddDimg(dimgPath string) error { + ds.storeLock.Lock() + defer ds.storeLock.Unlock() + dimgFile, err := OpenDimgFile(dimgPath) if err != nil { return fmt.Errorf("failed to open dimg %s: %v", dimgPath, err) @@ -91,7 +101,7 @@ func (ds *DimgStore) AddDimg(dimgPath string) error { DimgHeader: *header, Path: fPath, } - ds.dimgIds[header.Id] = entry + ds.dimgGraph.Add(header.ParentId.String(), header.Id.String(), header.Digest().String(), 1) ds.dimgDigests[header.Digest()] = entry return nil @@ -99,21 +109,29 @@ func (ds *DimgStore) AddDimg(dimgPath string) error { // string[0] == top // string[1] == layer(parentId top.Id) -func (ds *DimgStore) GetDimgPaths(dimgDigest digest.Digest) ([]string, error) { +func (ds *DimgStore) GetDimgPathsWithDimgDigest(dimgDigest digest.Digest) ([]string, error) { + ds.storeLock.Lock() + defer ds.storeLock.Unlock() + targetDimg, ok := ds.dimgDigests[dimgDigest] if !ok { return nil, fmt.Errorf("dimg for %s not found", dimgDigest) } - paths := []string{targetDimg.Path} - nextId := targetDimg.ParentId - for nextId != "" { - dimg, ok := ds.dimgIds[nextId] + _, dimgs, err := ds.dimgGraph.ShortestPath("", targetDimg.Id.String()) + if err != nil { + return nil, fmt.Errorf("failed to get shortest path for %s: %v", targetDimg.Id, err) + } + + paths := make([]string, len(dimgs)) + for i, dimgEdge := range dimgs { + dimg, ok := ds.dimgDigests[digest.Digest(dimgEdge.GetName())] if !ok { - return nil, fmt.Errorf("dimg for %s not found", dimgDigest) + return nil, fmt.Errorf("dimg for %s not found", dimgEdge.GetName()) } - paths = append(paths, dimg.Path) - nextId = dimg.ParentId + // dimgs are ordered from base to top + // we need to reverse the path + paths[len(dimgs)-i-1] = dimg.Path } return paths, nil