From 717e3b63f186f25d2bd3842233c46d37ec951899 Mon Sep 17 00:00:00 2001 From: Jared Allard Date: Mon, 4 Apr 2022 10:45:05 -0700 Subject: [PATCH] feat(stenciltest): introduce testing framework (#69) --- go.mod | 8 +- go.sum | 2 + internal/modules/modulestest/modulestest.go | 72 ++++++++++ manifest.yaml | 1 + pkg/stenciltest/stenciltest.go | 147 ++++++++++++++++++++ pkg/stenciltest/stenciltest_test.go | 68 +++++++++ pkg/stenciltest/testdata/args.tpl | 3 + pkg/stenciltest/testdata/error.tpl | 1 + pkg/stenciltest/testdata/test.tpl | 1 + 9 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 internal/modules/modulestest/modulestest.go create mode 100644 manifest.yaml create mode 100644 pkg/stenciltest/stenciltest.go create mode 100644 pkg/stenciltest/stenciltest_test.go create mode 100644 pkg/stenciltest/testdata/args.tpl create mode 100644 pkg/stenciltest/testdata/error.tpl create mode 100644 pkg/stenciltest/testdata/test.tpl diff --git a/go.mod b/go.mod index f9f5d945..9f12f78c 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,11 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +require ( + github.com/bradleyjkemp/cupaloy v2.3.0+incompatible + github.com/google/go-cmp v0.5.7 +) + require ( github.com/AlecAivazis/survey/v2 v2.3.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -37,6 +42,7 @@ require ( github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/danieljoos/wincred v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect @@ -45,7 +51,6 @@ require ( github.com/go-git/gcfg v1.5.0 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.7 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect @@ -69,6 +74,7 @@ require ( github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect diff --git a/go.sum b/go.sum index 52d9fb01..0578d10e 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bradleyjkemp/cupaloy v2.3.0+incompatible h1:UafIjBvWQmS9i/xRg+CamMrnLTKNzo+bdmT/oH34c2Y= +github.com/bradleyjkemp/cupaloy v2.3.0+incompatible/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= diff --git a/internal/modules/modulestest/modulestest.go b/internal/modules/modulestest/modulestest.go new file mode 100644 index 00000000..26a8332b --- /dev/null +++ b/internal/modules/modulestest/modulestest.go @@ -0,0 +1,72 @@ +// Copyright 2022 Outreach Corporation. All Rights Reserved. + +// Description: This file implements basic helpers for module +// test interaction + +// Package modulestest contains code for interacting with modules +// in tests. +package modulestest + +import ( + "context" + "io" + "os" + + "github.com/getoutreach/stencil/internal/modules" + "github.com/getoutreach/stencil/pkg/configuration" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// addTemplateToFS adds a template to a billy.Filesystem +func addTemplateToFS(fs billy.Filesystem, tpl string) error { + srcFile, err := os.Open(tpl) + if err != nil { + return errors.Wrapf(err, "failed to open template file %q", tpl) + } + defer srcFile.Close() + + destF, err := fs.Create(tpl) + if err != nil { + return errors.Wrapf(err, "failed to create template %q in memfs", tpl) + } + defer destF.Close() + + // Copy the template file to the fs + _, err = io.Copy(destF, srcFile) + return errors.Wrapf(err, "failed to copy template %q to memfs", tpl) +} + +// NewModuleFromTemplate creates a module with the provided template +// being the only file in the module. +func NewModuleFromTemplates(arguments map[string]configuration.Argument, templates ...string) (*modules.Module, error) { + fs := memfs.New() + for _, tpl := range templates { + if err := addTemplateToFS(fs, tpl); err != nil { + return nil, err + } + } + + mf, err := fs.Create("manifest.yaml") + if err != nil { + return nil, errors.Wrap(err, "failed to create in memory manifest file") + } + defer mf.Close() + + // write a manifest file so that we can handle arguments + enc := yaml.NewEncoder(mf) + if err := enc.Encode(&configuration.TemplateRepositoryManifest{ + Name: "modulestest", + Arguments: arguments, + }); err != nil { + return nil, errors.Wrap(err, "failed to encode generated module manifest") + } + if err := enc.Close(); err != nil { + return nil, errors.Wrap(err, "failed to close generated module manifest") + } + + // create the module + return modules.NewWithFS(context.Background(), "modulestest", fs), nil +} diff --git a/manifest.yaml b/manifest.yaml new file mode 100644 index 00000000..37a5aecc --- /dev/null +++ b/manifest.yaml @@ -0,0 +1 @@ +name: testing diff --git a/pkg/stenciltest/stenciltest.go b/pkg/stenciltest/stenciltest.go new file mode 100644 index 00000000..6ababb11 --- /dev/null +++ b/pkg/stenciltest/stenciltest.go @@ -0,0 +1,147 @@ +// Copyright 2022 Outreach Corporation. All Rights Reserved. + +// Description: This file implements the stenciltest framework +// for testing templates generated by stencil. + +// Package stenciltest contains code for testing templates +package stenciltest + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/bradleyjkemp/cupaloy" + "github.com/getoutreach/stencil/internal/codegen" + "github.com/getoutreach/stencil/internal/modules" + "github.com/getoutreach/stencil/internal/modules/modulestest" + "github.com/getoutreach/stencil/pkg/configuration" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "gotest.tools/v3/assert" +) + +// Template is a template that is being tested by the stenciltest framework. +type Template struct { + // path is the path to the template. + path string + + // aditionalTemplates is a list of additional templates to add to the renderer, + // but not to snapshot. + additionalTemplates []string + + // m is the template repository manifest for this test + m *configuration.TemplateRepositoryManifest + + // t is a testing object. + t *testing.T + + // args are the arguments to the template. + args map[string]interface{} + + // errStr is the string an error should contain, if this is set then the template + // MUST error. + errStr string + + // persist denotes if we should save a snapshot or not + // This is meant for tests. + persist bool +} + +// New creates a new test for a given template. +func New(t *testing.T, templatePath string, additionalTemplates ...string) *Template { + // GOMOD: /go.mod + b, err := exec.Command("go", "env", "GOMOD").Output() + if err != nil { + t.Fatalf("failed to determine path to manifest: %v", err) + } + basepath := strings.TrimSuffix(strings.TrimSpace(string(b)), "/go.mod") + + b, err = os.ReadFile(filepath.Join(basepath, "manifest.yaml")) + if err != nil { + t.Fatal(err) + } + + var m configuration.TemplateRepositoryManifest + if err := yaml.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + + return &Template{ + t: t, + m: &m, + path: templatePath, + additionalTemplates: additionalTemplates, + persist: true, + } +} + +// Args sets the arguments to the template. +func (t *Template) Args(args map[string]interface{}) *Template { + t.args = args + return t +} + +// ErrorContains denotes that this test run should fail, and the message +// should contain the provided string. +// +// t.ErrorContains("i am an error") +func (t *Template) ErrorContains(msg string) { + t.errStr = msg +} + +// Run runs the test. +func (t *Template) Run(save bool) { + t.t.Run(t.path, func(got *testing.T) { + m, err := modulestest.NewModuleFromTemplates(t.m.Arguments, append([]string{t.path}, t.additionalTemplates...)...) + if err != nil { + got.Fatalf("failed to create module from template %q", t.path) + } + + mf := &configuration.ServiceManifest{Name: "testing", Arguments: t.args, + Modules: []*configuration.TemplateRepository{{Name: m.Name}}} + st := codegen.NewStencil(mf, []*modules.Module{m}) + + tpls, err := st.Render(context.Background(), logrus.New()) + if err != nil { + if t.errStr != "" { + // if t.errStr was set then we expected an error, since that + // was set via t.ErrorContains() + if err == nil { + got.Fatal("expected error, got nil") + } + assert.ErrorContains(t.t, err, t.errStr, "expected render to fail with error containing %q", t.errStr) + } else { + got.Fatalf("failed to render: %v", err) + } + } + + for _, tpl := range tpls { + // skip templates that aren't the one we are testing + if tpl.Path != t.path { + continue + } + + for _, f := range tpl.Files { + // skip the snapshot + if !t.persist { + continue + } + + success := got.Run(f.Name(), func(got *testing.T) { + snapshot := cupaloy.New(cupaloy.ShouldUpdate(func() bool { return save }), cupaloy.CreateNewAutomatically(true)) + snapshot.SnapshotT(got, f) + }) + if !success { + got.Fatalf("Generated file %q did not match snapshot", f.Name()) + } + } + + // only ever process one template + break + } + }) +} diff --git a/pkg/stenciltest/stenciltest_test.go b/pkg/stenciltest/stenciltest_test.go new file mode 100644 index 00000000..8fe29b5f --- /dev/null +++ b/pkg/stenciltest/stenciltest_test.go @@ -0,0 +1,68 @@ +package stenciltest + +import ( + "testing" + + "github.com/getoutreach/stencil/pkg/configuration" + "github.com/google/go-cmp/cmp" + "gotest.tools/v3/assert" +) + +func TestMain(t *testing.T) { + st := &Template{ + path: "testdata/test.tpl", + additionalTemplates: make([]string, 0), + m: &configuration.TemplateRepositoryManifest{Name: "testing"}, + t: t, + persist: false, + } + st.Run(false) +} + +func TestErrorHandling(t *testing.T) { + st := &Template{ + path: "testdata/error.tpl", + additionalTemplates: make([]string, 0), + m: &configuration.TemplateRepositoryManifest{Name: "testing"}, + t: t, + persist: false, + } + st.ErrorContains("sad") + st.Run(false) + + st = &Template{ + path: "testdata/error.tpl", + additionalTemplates: make([]string, 0), + m: &configuration.TemplateRepositoryManifest{Name: "testing"}, + t: t, + persist: false, + } + st.ErrorContains("sad pikachu") + st.Run(false) +} + +func TestArgs(t *testing.T) { + st := &Template{ + path: "testdata/args.tpl", + additionalTemplates: make([]string, 0), + m: &configuration.TemplateRepositoryManifest{Name: "testing", Arguments: map[string]configuration.Argument{ + "hello": { + Type: "string", + }, + }}, + t: t, + persist: false, + } + st.Args(map[string]interface{}{"hello": "world"}) + st.Run(false) +} + +// Doing this just to bump up coverage numbers, we essentially test this w/ the Template +// constructors in each test. +func TestCoverageHack(t *testing.T) { + st := New(t, "testdata/test.tpl") + assert.Equal(t, st.path, "testdata/test.tpl") + assert.Equal(t, st.persist, true) + assert.Assert(t, !cmp.Equal(st.t, nil)) + assert.Equal(t, st.m.Name, "testing") +} diff --git a/pkg/stenciltest/testdata/args.tpl b/pkg/stenciltest/testdata/args.tpl new file mode 100644 index 00000000..d0b2fce6 --- /dev/null +++ b/pkg/stenciltest/testdata/args.tpl @@ -0,0 +1,3 @@ +{{ if ne (stencil.Arg "hello") "world" }} +{{ fail "expected .hello to be 'world' "}} +{{ end }} diff --git a/pkg/stenciltest/testdata/error.tpl b/pkg/stenciltest/testdata/error.tpl new file mode 100644 index 00000000..b9fb9502 --- /dev/null +++ b/pkg/stenciltest/testdata/error.tpl @@ -0,0 +1 @@ +{{ fail "sad pikachu" }} diff --git a/pkg/stenciltest/testdata/test.tpl b/pkg/stenciltest/testdata/test.tpl new file mode 100644 index 00000000..d8e31cfb --- /dev/null +++ b/pkg/stenciltest/testdata/test.tpl @@ -0,0 +1 @@ +{{ "hello, world!" }}