feat(stenciltest): introduce testing framework (#69)

go.mod

@@ -24,6 +24,11 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
+require (
+	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
+	github.com/google/go-cmp v0.5.7
+)
+
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.2 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
@@ -37,6 +42,7 @@ require (
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/danieljoos/wincred v1.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect
@@ -45,7 +51,6 @@ require (
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/godbus/dbus/v5 v5.0.4 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
@@ -69,6 +74,7 @@ require (
 	github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
 	github.com/mitchellh/reflectwalk v1.0.0 // indirect
 	github.com/oklog/run v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect

internal/modules/modulestest/modulestest.go

@@ -0,0 +1,72 @@
+// Copyright 2022 Outreach Corporation. All Rights Reserved.
+
+// Description: This file implements basic helpers for module
+// test interaction
+
+// Package modulestest contains code for interacting with modules
+// in tests.
+package modulestest
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/getoutreach/stencil/internal/modules"
+	"github.com/getoutreach/stencil/pkg/configuration"
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+// addTemplateToFS adds a template to a billy.Filesystem
+func addTemplateToFS(fs billy.Filesystem, tpl string) error {
+	srcFile, err := os.Open(tpl)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open template file %q", tpl)
+	}
+	defer srcFile.Close()
+
+	destF, err := fs.Create(tpl)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create template %q in memfs", tpl)
+	}
+	defer destF.Close()
+
+	// Copy the template file to the fs
+	_, err = io.Copy(destF, srcFile)
+	return errors.Wrapf(err, "failed to copy template %q to memfs", tpl)
+}
+
+// NewModuleFromTemplate creates a module with the provided template
+// being the only file in the module.
+func NewModuleFromTemplates(arguments map[string]configuration.Argument, templates ...string) (*modules.Module, error) {
+	fs := memfs.New()
+	for _, tpl := range templates {
+		if err := addTemplateToFS(fs, tpl); err != nil {
+			return nil, err
+		}
+	}
+
+	mf, err := fs.Create("manifest.yaml")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create in memory manifest file")
+	}
+	defer mf.Close()
+
+	// write a manifest file so that we can handle arguments
+	enc := yaml.NewEncoder(mf)
+	if err := enc.Encode(&configuration.TemplateRepositoryManifest{
+		Name:      "modulestest",
+		Arguments: arguments,
+	}); err != nil {
+		return nil, errors.Wrap(err, "failed to encode generated module manifest")
+	}
+	if err := enc.Close(); err != nil {
+		return nil, errors.Wrap(err, "failed to close generated module manifest")
+	}
+
+	// create the module
+	return modules.NewWithFS(context.Background(), "modulestest", fs), nil
+}

manifest.yaml

@@ -0,0 +1 @@
+name: testing

pkg/stenciltest/stenciltest.go

@@ -0,0 +1,147 @@
+// Copyright 2022 Outreach Corporation. All Rights Reserved.
+
+// Description: This file implements the stenciltest framework
+// for testing templates generated by stencil.
+
+// Package stenciltest contains code for testing templates
+package stenciltest
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/bradleyjkemp/cupaloy"
+	"github.com/getoutreach/stencil/internal/codegen"
+	"github.com/getoutreach/stencil/internal/modules"
+	"github.com/getoutreach/stencil/internal/modules/modulestest"
+	"github.com/getoutreach/stencil/pkg/configuration"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+	"gotest.tools/v3/assert"
+)
+
+// Template is a template that is being tested by the stenciltest framework.
+type Template struct {
+	// path is the path to the template.
+	path string
+
+	// aditionalTemplates is a list of additional templates to add to the renderer,
+	// but not to snapshot.
+	additionalTemplates []string
+
+	// m is the template repository manifest for this test
+	m *configuration.TemplateRepositoryManifest
+
+	// t is a testing object.
+	t *testing.T
+
+	// args are the arguments to the template.
+	args map[string]interface{}
+
+	// errStr is the string an error should contain, if this is set then the template
+	// MUST error.
+	errStr string
+
+	// persist denotes if we should save a snapshot or not
+	// This is meant for tests.
+	persist bool
+}
+
+// New creates a new test for a given template.
+func New(t *testing.T, templatePath string, additionalTemplates ...string) *Template {
+	// GOMOD: <module path>/go.mod
+	b, err := exec.Command("go", "env", "GOMOD").Output()
+	if err != nil {
+		t.Fatalf("failed to determine path to manifest: %v", err)
+	}
+	basepath := strings.TrimSuffix(strings.TrimSpace(string(b)), "/go.mod")
+
+	b, err = os.ReadFile(filepath.Join(basepath, "manifest.yaml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var m configuration.TemplateRepositoryManifest
+	if err := yaml.Unmarshal(b, &m); err != nil {
+		t.Fatal(err)
+	}
+
+	return &Template{
+		t:                   t,
+		m:                   &m,
+		path:                templatePath,
+		additionalTemplates: additionalTemplates,
+		persist:             true,
+	}
+}
+
+// Args sets the arguments to the template.
+func (t *Template) Args(args map[string]interface{}) *Template {
+	t.args = args
+	return t
+}
+
+// ErrorContains denotes that this test run should fail, and the message
+// should contain the provided string.
+//
+//   t.ErrorContains("i am an error")
+func (t *Template) ErrorContains(msg string) {
+	t.errStr = msg
+}
+
+// Run runs the test.
+func (t *Template) Run(save bool) {
+	t.t.Run(t.path, func(got *testing.T) {
+		m, err := modulestest.NewModuleFromTemplates(t.m.Arguments, append([]string{t.path}, t.additionalTemplates...)...)
+		if err != nil {
+			got.Fatalf("failed to create module from template %q", t.path)
+		}
+
+		mf := &configuration.ServiceManifest{Name: "testing", Arguments: t.args,
+			Modules: []*configuration.TemplateRepository{{Name: m.Name}}}
+		st := codegen.NewStencil(mf, []*modules.Module{m})
+
+		tpls, err := st.Render(context.Background(), logrus.New())
+		if err != nil {
+			if t.errStr != "" {
+				// if t.errStr was set then we expected an error, since that
+				// was set via t.ErrorContains()
+				if err == nil {
+					got.Fatal("expected error, got nil")
+				}
+				assert.ErrorContains(t.t, err, t.errStr, "expected render to fail with error containing %q", t.errStr)
+			} else {
+				got.Fatalf("failed to render: %v", err)
+			}
+		}
+
+		for _, tpl := range tpls {
+			// skip templates that aren't the one we are testing
+			if tpl.Path != t.path {
+				continue
+			}
+
+			for _, f := range tpl.Files {
+				// skip the snapshot
+				if !t.persist {
+					continue
+				}
+
+				success := got.Run(f.Name(), func(got *testing.T) {
+					snapshot := cupaloy.New(cupaloy.ShouldUpdate(func() bool { return save }), cupaloy.CreateNewAutomatically(true))
+					snapshot.SnapshotT(got, f)
+				})
+				if !success {
+					got.Fatalf("Generated file %q did not match snapshot", f.Name())
+				}
+			}
+
+			// only ever process one template
+			break
+		}
+	})
+}

pkg/stenciltest/stenciltest_test.go

@@ -0,0 +1,68 @@
+package stenciltest
+
+import (
+	"testing"
+
+	"github.com/getoutreach/stencil/pkg/configuration"
+	"github.com/google/go-cmp/cmp"
+	"gotest.tools/v3/assert"
+)
+
+func TestMain(t *testing.T) {
+	st := &Template{
+		path:                "testdata/test.tpl",
+		additionalTemplates: make([]string, 0),
+		m:                   &configuration.TemplateRepositoryManifest{Name: "testing"},
+		t:                   t,
+		persist:             false,
+	}
+	st.Run(false)
+}
+
+func TestErrorHandling(t *testing.T) {
+	st := &Template{
+		path:                "testdata/error.tpl",
+		additionalTemplates: make([]string, 0),
+		m:                   &configuration.TemplateRepositoryManifest{Name: "testing"},
+		t:                   t,
+		persist:             false,
+	}
+	st.ErrorContains("sad")
+	st.Run(false)
+
+	st = &Template{
+		path:                "testdata/error.tpl",
+		additionalTemplates: make([]string, 0),
+		m:                   &configuration.TemplateRepositoryManifest{Name: "testing"},
+		t:                   t,
+		persist:             false,
+	}
+	st.ErrorContains("sad pikachu")
+	st.Run(false)
+}
+
+func TestArgs(t *testing.T) {
+	st := &Template{
+		path:                "testdata/args.tpl",
+		additionalTemplates: make([]string, 0),
+		m: &configuration.TemplateRepositoryManifest{Name: "testing", Arguments: map[string]configuration.Argument{
+			"hello": {
+				Type: "string",
+			},
+		}},
+		t:       t,
+		persist: false,
+	}
+	st.Args(map[string]interface{}{"hello": "world"})
+	st.Run(false)
+}
+
+// Doing this just to bump up coverage numbers, we essentially test this w/ the Template
+// constructors in each test.
+func TestCoverageHack(t *testing.T) {
+	st := New(t, "testdata/test.tpl")
+	assert.Equal(t, st.path, "testdata/test.tpl")
+	assert.Equal(t, st.persist, true)
+	assert.Assert(t, !cmp.Equal(st.t, nil))
+	assert.Equal(t, st.m.Name, "testing")
+}

pkg/stenciltest/testdata/args.tpl

@@ -0,0 +1,3 @@
+{{ if ne (stencil.Arg "hello") "world" }}
+{{ fail "expected .hello to be 'world' "}}
+{{ end }}

pkg/stenciltest/testdata/error.tpl

@@ -0,0 +1 @@
+{{ fail "sad pikachu" }}

pkg/stenciltest/testdata/test.tpl

@@ -0,0 +1 @@
+{{ "hello, world!" }}