Skip to content

Commit

Permalink
[backport] feat(go/vulndb): support the new OSV format (#165)
Browse files Browse the repository at this point in the history 
* chore(mod): tidy (#163)

* refactor: use WalkDir

* feat(go/vulndb): support a new OSV format
  • Loading branch information
@knqyf263
knqyf263 authored Dec 20, 2021
1 parent 1183296 commit be59a21
Show file tree
Hide file tree
Showing 10 changed files with 777 additions and 233 deletions.
7 changes: 4 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,18 @@ go 1.15
require (
github.com/BurntSushi/toml v0.3.1
github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986
github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2
github.com/briandowns/spinner v1.12.0
github.com/fatih/color v1.10.0
github.com/google/go-github/v38 v38.1.0
github.com/hashicorp/go-version v1.2.1
github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
github.com/stretchr/testify v1.6.1
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/stretchr/testify v1.7.0
github.com/urfave/cli v1.22.5
go.etcd.io/bbolt v1.3.5
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
golang.org/x/vuln v0.0.0-20211215213114-5e054cb3e47e
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
gopkg.in/cheggaaa/pb.v1 v1.0.28
gopkg.in/yaml.v2 v2.4.0
Expand Down
582 changes: 545 additions & 37 deletions go.sum
View file

Large diffs are not rendered by default.

11 changes: 8 additions & 3 deletions pkg/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package utils
import (
"bytes"
"io"
"io/fs"
"log"
"os"
"os/exec"
Expand All @@ -22,14 +23,18 @@ func CacheDir() string {
}

func FileWalk(root string, walkFn func(r io.Reader, path string) error) error {
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if info.IsDir() {
} else if d.IsDir() {
return nil
}

info, err := d.Info()
if err != nil {
return xerrors.Errorf("file info error: %w", err)
}

if info.Size() == 0 {
log.Printf("invalid size: %s\n", path)
return nil
Expand Down
111 changes: 75 additions & 36 deletions pkg/vulnsrc/govulndb/govulndb.go
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package govulndb

import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"path/filepath"
"strings"

bolt "go.etcd.io/bbolt"
"golang.org/x/vuln/osv"
"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/db"
Expand Down Expand Up @@ -41,16 +42,16 @@ func (vs VulnSrc) Update(dir string) error {
rootDir := filepath.Join(dir, "vuln-list", govulndbDir)

var items []Entry
buffer := &bytes.Buffer{}
err := utils.FileWalk(rootDir, func(r io.Reader, path string) error {
item := Entry{}
if _, err := buffer.ReadFrom(r); err != nil {
return xerrors.Errorf("failed to read file (%s): %w", path, err)
var item Entry
if err := json.NewDecoder(r).Decode(&item); err != nil {
return xerrors.Errorf("JSON decode error (%s): %w", path, err)
}
if err := json.Unmarshal(buffer.Bytes(), &item); err != nil {
return xerrors.Errorf("JSON error (%s): %w", path, err)

// Standard libraries are not listed in go.sum, etc.
if item.Module == "stdlib" {
return nil
}
buffer.Reset()
items = append(items, item)
return nil
})
Expand Down Expand Up @@ -89,41 +90,52 @@ func (vs VulnSrc) commit(tx *bolt.Tx, item Entry) error {
vulnIDs = []string{item.ID}
}

var patchedVersions, vulnerableVersions []string
for _, affect := range item.Affects.Ranges {
// patched versions
patchedVersions = append(patchedVersions, affect.Fixed)
// Take a single affected entry
affected := findAffected(item.Module, item.Affected)
if len(affected.Ranges) == 0 {
return xerrors.Errorf("invalid entry: %s %s", item.ID, item.Module)
}

if affect.Fixed == "" {
continue
var patchedVersions, vulnerableVersions, references []string
for _, affects := range affected.Ranges {
var vulnerable string
for _, event := range affects.Events {
switch {
case event.Introduced != "":
// e.g. {"introduced": "1.2.0}, {"introduced": "2.2.0}
if vulnerable != "" {
vulnerableVersions = append(vulnerableVersions, vulnerable)
}
vulnerable = fmt.Sprintf(">=%s", event.Introduced)
case event.Fixed != "":
// patched versions
patchedVersions = append(patchedVersions, event.Fixed)

// e.g. {"introduced": "1.2.0}, {"fixed": "1.2.5}
vulnerable = fmt.Sprintf("%s, <%s", vulnerable, event.Fixed)
}
}

// vulnerable versions
vulnerable := fmt.Sprintf("< %s", affect.Fixed)
if affect.Introduced != "" {
vulnerable = fmt.Sprintf(">= %s, %s", affect.Introduced, vulnerable)
if vulnerable != "" {
vulnerableVersions = append(vulnerableVersions, vulnerable)
}
}

if affected.DatabaseSpecific.URL != "" {
references = append(references, affected.DatabaseSpecific.URL)
}

vulnerableVersions = append(vulnerableVersions, vulnerable)
// Update references
for _, ref := range item.References {
references = append(references, ref.URL)
}

a := types.Advisory{
PatchedVersions: patchedVersions,
VulnerableVersions: vulnerableVersions,
}

// A module name must be filled.
pkgName := item.Module
if pkgName == "" {
pkgName = item.Package.Name
}

var references []string
for _, ref := range item.References {
references = append(references, ref.URL)
}
if item.EcosystemSpecific.URL != "" {
references = append(references, item.EcosystemSpecific.URL)
}

prefixedBucketName, _ := bucket.Name(vulnerability.Go, bucketName)
for _, vulnID := range vulnIDs {
Expand All @@ -133,11 +145,8 @@ func (vs VulnSrc) commit(tx *bolt.Tx, item Entry) error {
}

vuln := types.VulnerabilityDetail{
ID: vulnID,
Description: item.Details,
References: references,
PublishedDate: &item.Published,
LastModifiedDate: &item.Modified,
Description: item.Details,
References: references,
}
if err = vs.dbc.PutVulnerabilityDetail(tx, vulnID, prefixedBucketName, vuln); err != nil {
return xerrors.Errorf("failed to put vulnerability detail (%s): %w", vulnID, err)
Expand All @@ -150,3 +159,33 @@ func (vs VulnSrc) commit(tx *bolt.Tx, item Entry) error {

return nil
}

func findAffected(module string, affectedList []osv.Affected) osv.Affected {
// Multiple packages may be included in "affected".
// We have to select the appropriate package matching the module name
// because those packages may have different versioning.
//
// e.g. GO-2020-0017
// module => github.com/dgrijalva/jwt-go/v4
// packages => github.com/dgrijalva/jwt-go and github.com/dgrijalva/jwt-go/v4
//
// We should ignore "github.com/dgrijalva/jwt-go" in the above case.
for _, a := range affectedList {
if a.Package.Name == module {
return a
}
}

// If there is no package that exactly matches the module name,
// we'll choose a package that contains the module name.
// e.g. GO-2021-0101
// module => github.com/apache/thrift
// package => github.com/apache/thrift/lib/go/thrift
for _, a := range affectedList {
if strings.HasPrefix(a.Package.Name, module) {
return a
}
}

return osv.Affected{}
}
48 changes: 17 additions & 31 deletions pkg/vulnsrc/govulndb/govulndb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy-db/pkg/dbtest"
"github.com/aquasecurity/trivy-db/pkg/types"
"github.com/aquasecurity/trivy-db/pkg/utils"
"github.com/aquasecurity/trivy-db/pkg/vulnsrc/govulndb"
)

Expand All @@ -29,60 +28,47 @@ func TestVulnSrc_Update(t *testing.T) {
dir: "testdata/happy",
want: []wantKV{
{
key: []string{"advisory-detail", "CVE-2020-29242", "go::vulndb", "github.com/dhowden/tag"},
key: []string{"advisory-detail", "CVE-2019-0210", "go::vulndb", "github.com/apache/thrift"},
value: types.Advisory{
PatchedVersions: []string{"v0.0.0-20201120070457-d52dcb253c63"},
VulnerableVersions: []string{"< v0.0.0-20201120070457-d52dcb253c63"},
PatchedVersions: []string{"0.13.0"},
VulnerableVersions: []string{">=0.0.0-20151001171628-53dd39833a08, <0.13.0"},
},
},
{
key: []string{"vulnerability-detail", "CVE-2020-29242", "go::vulndb"},
key: []string{"vulnerability-detail", "CVE-2019-0210", "go::vulndb"},
value: types.VulnerabilityDetail{
ID: "CVE-2020-29242",
Description: "Due to improper bounds checking a number of methods can trigger a panic due to attempted\nout-of-bounds reads. If the package is used to parse user supplied input this may be\nused as a vector for a denial of service attack.\n",
Description: "Due to an improper bounds check, parsing maliciously crafted messages can cause panics. If\nthis package is used to parse untrusted input, this may be used as a vector for a denial of\nservice attack.\n",
References: []string{
"https://github.com/dhowden/tag/commit/d52dcb253c63a153632bfee5f269dd411dcd8e96",
"https://github.com/dhowden/tag/commit/a92213460e4838490ce3066ef11dc823cdc1740e",
"https://github.com/dhowden/tag/commit/4b595ed4fac79f467594aa92f8953f90f817116e",
"https://github.com/dhowden/tag/commit/6b18201aa5c5535511802ddfb4e4117686b4866d",
"https://go.googlesource.com/vulndb/+/refs/heads/main/reports/GO-2021-0097.toml",
"https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2021-0101.yaml",
"https://github.com/apache/thrift/commit/264a3f318ed3e9e51573f67f963c8509786bcec2",
"https://github.com/advisories/GHSA-jq7p-26h5-w78r",
},
PublishedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
LastModifiedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
},
},
},
},
{
name: "missing module",
dir: "testdata/no-module",
want: []wantKV{
{
key: []string{"advisory-detail", "GO-2021-0090", "go::vulndb", "github.com/tendermint/tendermint/types"},
key: []string{"advisory-detail", "CVE-2020-26160", "go::vulndb", "github.com/dgrijalva/jwt-go/v4"},
value: types.Advisory{
PatchedVersions: []string{"v0.34.0-dev1.0.20200702134149-480b995a3172"},
VulnerableVersions: []string{">= v0.33.0, < v0.34.0-dev1.0.20200702134149-480b995a3172"},
PatchedVersions: []string{"4.0.0-preview1"},
VulnerableVersions: []string{">=0, <4.0.0-preview1"},
},
},
{
key: []string{"vulnerability-detail", "GO-2021-0090", "go::vulndb"},
key: []string{"vulnerability-detail", "CVE-2020-26160", "go::vulndb"},
value: types.VulnerabilityDetail{
ID: "GO-2021-0090",
Description: "Proposed commits may contain signatures for blocks not contained within the commit. Instead of skipping\nthese signatures, they cause failure during verification. A malicious proposer can use this to force\nconsensus failures.\n",
Description: "If a JWT contains an audience claim with an array of strings, rather\nthan a single string, and `MapClaims.VerifyAudience` is called with\n`req` set to `false`, then audience verification will be bypassed,\nallowing an invalid set of audiences to be provided.\n",
References: []string{
"https://github.com/tendermint/tendermint/pull/5426",
"https://go.googlesource.com/vulndb/+/refs/heads/main/reports/GO-2021-0090.toml",
"https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2020-0017.yaml",
"https://github.com/dgrijalva/jwt-go/commit/ec0a89a131e3e8567adcb21254a5cd20a70ea4ab",
"https://github.com/dgrijalva/jwt-go/issues/422",
},
PublishedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
LastModifiedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
},
},
},
},
{
name: "broken JSON",
dir: "testdata/broken",
wantErr: "JSON error",
wantErr: "JSON decode error",
},
{
name: "sad path",
Expand Down
73 changes: 73 additions & 0 deletions pkg/vulnsrc/govulndb/testdata/happy/vuln-list/go/GO-2020-0017.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"module": "github.com/dgrijalva/jwt-go/v4",
"id": "GO-2020-0017",
"published": "2021-04-14T12:00:00Z",
"modified": "2021-04-14T12:00:00Z",
"aliases": [
"CVE-2020-26160"
],
"details": "If a JWT contains an audience claim with an array of strings, rather\nthan a single string, and `MapClaims.VerifyAudience` is called with\n`req` set to `false`, then audience verification will be bypassed,\nallowing an invalid set of audiences to be provided.\n",
"affected": [
{
"package": {
"name": "github.com/dgrijalva/jwt-go",
"ecosystem": "Go"
},
"ranges": [
{
"type": "SEMVER",
"events": [
{
"introduced": "0.0.0-20150717181359-44718f8a89b0"
}
]
}
],
"database_specific": {
"url": "https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2020-0017.yaml"
},
"ecosystem_specific": {
"symbols": [
"MapClaims.VerifyAudience"
]
}
},
{
"package": {
"name": "github.com/dgrijalva/jwt-go/v4",
"ecosystem": "Go"
},
"ranges": [
{
"type": "SEMVER",
"events": [
{
"introduced": "0"
},
{
"fixed": "4.0.0-preview1"
}
]
}
],
"database_specific": {
"url": "https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2020-0017.yaml"
},
"ecosystem_specific": {
"symbols": [
"MapClaims.VerifyAudience"
]
}
}
],
"references": [
{
"type": "FIX",
"url": "https://github.com/dgrijalva/jwt-go/commit/ec0a89a131e3e8567adcb21254a5cd20a70ea4ab"
},
{
"type": "WEB",
"url": "https://github.com/dgrijalva/jwt-go/issues/422"
}
]
}
Loading

0 comments on commit be59a21

Please sign in to comment.