Buildpack metadata is added by deplab

* reads from label io.buildpacks.build.metadata
* integration tests provided
* buildpack metadata and buildpack bom metadata is ordered
  alphabetically by id and name, respectively.
* sha256 of json encoded metadata is calculated; this matches behaviour of dpkg and rpm lists
* refactors Digest to common package; removes duplication from dpkg, rpm and
  cnb
* adds integration test for inspect a scratch image
* adds a scratch image test asset
* created more tests for merge
* amends documentation to better explain the revised behaviour of
inspect

[#170631159]

Signed-off-by: Iain Sproat <[email protected]>

README.md

@@ -10,7 +10,9 @@ Download the latest deplab release matching your OS from https://github.com/pivo
 
 By default `deplab` [generates](#generate-metadata) the metadata of an image and the provided git repository (from where the image is built). The metadata is placed in a label on the output image, which are read by Pivotal's Open Source Licensing (OSL) process when using the container_image scan root. Once an image is labelled with `deplab` the metadata can be visualized using [inspect](#inspect).
 
-Deplab currently supports the auto-generation of dpkg and rpm package lists. Additional sources can be entered manually. RPM support is currently experimental.  It necessitates the presence of the `rpm` binary in the  `$PATH` where deplab is run.
+`deplab` currently supports the auto-generation of dpkg and rpm package lists.  RPM support is currently experimental.  It necessitates the presence of the `rpm` binary in the  `$PATH` where `deplab` is run.  Additional sources can be entered manually.
+
+If the image being inspected was created by Cloud Native Buildpacks, `deplab` will report the buildpack build metadata found on the `io.buildpacks.build.metadata` label on the image. 
 
 To generate the metadata and output a labelled image, run
 ```bash
@@ -50,7 +52,9 @@ Then, to visualise the metadata (optional), run
 |  | `--version` |  |  version for deplab |  | 
 
 ## Inspect
-Inspect is used to view the deplab metadata on a container image. Inspect prints the deplab "io.pivotal.metadata" label in the config file of an image to stdout.  The label will be printed in JSON format. If metadata does not exist on the image an error will be printed to standard error.
+Inspect reports on the content of the container image and also validates any existing deplab labels that it finds on the image. It will additionally show elements of the deplab label on the image that cannot be generated by inspecting the image, e.g. git and archive urls. Inspect prints its findings to stdout and reports any validation warnings to stderr.  The label will be printed in JSON format.
+
+The inspect command can be used on both images which have been previously labelled by deplab and images which have not.
 
 `deplab inspect` requires one image source to be specified (`--image` or `--image-tar`).
 

go.sum

@@ -1,6 +1,7 @@
 cloud.google.com/go v0.25.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/azure-sdk-for-go v19.1.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -92,8 +93,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
-github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=

pkg/cnb/provider.go

@@ -0,0 +1,80 @@
+package cnb
+
+import (
+	"encoding/json"
+	"sort"
+
+	"github.com/pivotal/deplab/pkg/common"
+
+	"github.com/pivotal/deplab/pkg/metadata"
+	"github.com/pkg/errors"
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
+
+	"github.com/pivotal/deplab/pkg/image"
+)
+
+func Provider(dli image.Image, _ common.RunParams, md metadata.Metadata) (metadata.Metadata, error) {
+	dependency, err := buildDependencyMetadata(dli)
+	if err != nil {
+		return metadata.Metadata{}, err
+	}
+	md.Dependencies = append(md.Dependencies, dependency)
+	return md, nil
+}
+
+func buildDependencyMetadata(dli image.Image) (metadata.Dependency, error) {
+	var buildpackMetadataContents string
+	config, err := dli.GetConfig()
+
+	if err != nil {
+		return metadata.Dependency{}, err
+	}
+
+	buildpackMetadataContents = config.Config.Labels["io.buildpacks.build.metadata"]
+
+	if buildpackMetadataContents != "" {
+		buildpackMetadata, err := parseMetadataJSON(buildpackMetadataContents)
+		if err != nil {
+			return metadata.Dependency{}, errors.Wrapf(err, "Could not parse buildpack metadata toml")
+		}
+
+		version, err := common.Digest(buildpackMetadata)
+		if err != nil {
+			return metadata.Dependency{}, errors.Wrapf(err, "Could not get digest for buildpack metadata")
+		}
+
+		return metadata.Dependency{
+			Type: metadata.BuildpackMetadataType,
+			Source: metadata.Source{
+				Type:     "inline",
+				Metadata: buildpackMetadata,
+				Version: map[string]interface{}{
+					"sha256": version,
+				},
+			},
+		}, nil
+	}
+
+	return metadata.Dependency{}, nil
+}
+
+func parseMetadataJSON(buildpackMetadata string) (metadata.BuildpackBOMSourceMetadata, error) {
+	var bp metadata.BuildpackBOMSourceMetadata
+
+	err := json.Unmarshal([]byte(buildpackMetadata), &bp)
+	if err != nil {
+		return metadata.BuildpackBOMSourceMetadata{}, errors.Wrapf(err, "could not decode json")
+	}
+
+	collator := collate.New(language.BritishEnglish)
+	sort.Slice(bp.Buildpacks, func(i, j int) bool {
+		return collator.CompareString(bp.Buildpacks[i].ID, bp.Buildpacks[j].ID) < 0
+	})
+
+	sort.Slice(bp.BillOfMaterials, func(i, j int) bool {
+		return collator.CompareString(bp.BillOfMaterials[i].Name, bp.BillOfMaterials[j].Name) < 0
+	})
+
+	return bp, nil
+}

pkg/common/common.go

@@ -1,5 +1,13 @@
 package common
 
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
 type RunParams struct {
 	InputImageTarPath         string
 	InputImage                string
@@ -12,3 +20,14 @@ type RunParams struct {
 	AdditionalSourceFilePaths []string
 	IgnoreValidationErrors    bool
 }
+
+func Digest(sourceMetadata interface{}) (string, error) {
+	hash := sha256.New()
+	encoder := json.NewEncoder(hash)
+	err := encoder.Encode(sourceMetadata)
+	if err != nil {
+		return "", errors.Wrapf(err, "could not encode source metadata.")
+	}
+	version := hex.EncodeToString(hash.Sum(nil))
+	return version, nil
+}

pkg/common/common_suite_test.go

@@ -0,0 +1,13 @@
+package common_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCommon(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Common Suite")
+}

pkg/common/common_test.go

@@ -0,0 +1,137 @@
+package common_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	. "github.com/pivotal/deplab/pkg/common"
+	"github.com/pivotal/deplab/pkg/metadata"
+)
+
+var _ = Describe("Digest", func() {
+	It("generate a sha256 of the input", func() {
+		out, err := Digest(metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "foo",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+			},
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(out).To(Equal("7ecf7aa2c71ee01ec2b90f37a3b8e944158e9aea6b8cee0290a7cb187884cf4c"))
+	})
+
+	It("generates the same digest for 2 different input instances with the same content", func() {
+		input1 := metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "foo",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+			},
+		}
+		input2 := metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "foo",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+			},
+		}
+
+		out1, err := Digest(input1)
+		Expect(err).ToNot(HaveOccurred())
+		out2, err := Digest(input2)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(&input1 != &input2).To(BeTrue())
+		Expect(out2).To(Equal(out1))
+	})
+
+	It("generates a different digest for 2 different input instances with the same content in different order", func() {
+		input1 := metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "foo",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+				{
+					Package: "bar",
+					Source: metadata.PackageSource{
+						Version: "1.0",
+					},
+				},
+			},
+		}
+		input2 := metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "bar",
+					Source: metadata.PackageSource{
+						Version: "1.0",
+					},
+				},
+				{
+					Package: "foo",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+			},
+		}
+
+		out1, err := Digest(input1)
+		Expect(err).ToNot(HaveOccurred())
+		out2, err := Digest(input2)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(&input1 != &input2).To(BeTrue())
+		Expect(out2).ToNot(Equal(out1))
+	})
+
+	It("generates different digest for 2 different inputs", func() {
+		input1 := metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "bar",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+			},
+		}
+		input2 := metadata.DebianPackageListSourceMetadata{
+			AptSources: []string{"deb example.com bionic main universe"},
+			Packages: []metadata.DpkgPackage{
+				{
+					Package: "foo",
+					Source: metadata.PackageSource{
+						Version: "4.2",
+					},
+				},
+			},
+		}
+
+		out1, err := Digest(input1)
+		Expect(err).ToNot(HaveOccurred())
+		out2, err := Digest(input2)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(out2).ToNot(Equal(out1))
+	})
+})

pkg/deplab/deplab.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/pivotal/deplab/pkg/cnb"
+
 	"github.com/pivotal/deplab/pkg/additionalsources"
 	"github.com/pivotal/deplab/pkg/common"
 	"github.com/pivotal/deplab/pkg/rpm"
@@ -46,6 +48,7 @@ func Run(params common.RunParams) error {
 	for _, provider := range []provider{
 		dpkg.Provider,
 		rpm.Provider,
+		cnb.Provider,
 		git.Provider,
 		additionalsources.ArchiveUrlProvider,
 		additionalsources.AdditionalSourcesProvider,
@@ -79,6 +82,7 @@ func RunInspect(inputImage, inputImageTar string) error {
 	for _, provider := range []provider{
 		dpkg.Provider,
 		rpm.Provider,
+		cnb.Provider,
 		osrelease.Provider,
 		ProvenanceProvider,
 		ExistingLabelProvider,

pkg/dpkg/provider.go

@@ -1,9 +1,6 @@
 package dpkg
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"sort"
 	"strings"
 
@@ -42,7 +39,7 @@ func BuildDependencyMetadata(dli image.Image) (metadata.Dependency, error) {
 			AptSources: sources,
 		}
 
-		version, err := Digest(sourceMetadata)
+		version, err := common.Digest(sourceMetadata)
 		if err != nil {
 			return metadata.Dependency{}, errors.Wrapf(err, "Could not get digest for source metadata")
 		}
@@ -64,17 +61,6 @@ func BuildDependencyMetadata(dli image.Image) (metadata.Dependency, error) {
 	return metadata.Dependency{}, err
 }
 
-func Digest(sourceMetadata metadata.DebianPackageListSourceMetadata) (string, error) {
-	hash := sha256.New()
-	encoder := json.NewEncoder(hash)
-	err := encoder.Encode(sourceMetadata)
-	if err != nil {
-		return "", errors.Wrapf(err, "could not encode source metadata.")
-	}
-	version := hex.EncodeToString(hash.Sum(nil))
-	return version, nil
-}
-
 func getAptSources(dli image.Image) ([]string, error) {
 	sources := []string{}
 	fileListContent, err := dli.GetDirContents("/etc/apt/sources.list.d")