feat: support prerelease on module (DTSS-1924) (#101)

Co-authored-by: George Shaw <[email protected]>
Co-authored-by: Chris Robinson <[email protected]>

cmd/stencil/stencil.go

@@ -83,7 +83,8 @@ func main() {
 				}
 			}
 
-			cmd := stencil.NewCommand(log, serviceManifest, c.Bool("dry-run"), c.Bool("frozen-lockfile"))
+			cmd := stencil.NewCommand(log, serviceManifest, c.Bool("dry-run"),
+				c.Bool("frozen-lockfile"), c.Bool("use-prerelease"))
 			return errors.Wrap(cmd.Run(ctx), "run codegen")
 		},
 		///EndBlock(app)
@@ -99,6 +100,10 @@ func main() {
 			Name:  "frozen-lockfile",
 			Usage: "Use versions from the lockfile instead of the latest",
 		},
+		&cli.BoolFlag{
+			Name:  "use-prerelease",
+			Usage: "Use prerelease versions of stencil modules",
+		},
 		///EndBlock(flags)
 	}
 	app.Commands = []*cli.Command{

go.mod

@@ -28,6 +28,7 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/google/go-cmp v0.5.8
+	github.com/google/go-github/v43 v43.0.0
 )
 
 require (
@@ -54,7 +55,6 @@ require (
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-github/v43 v43.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect

internal/cmd/stencil/stencil.go

@@ -51,12 +51,19 @@ type Command struct {
 }
 
 // NewCommand creates a new stencil command
-func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun, frozen bool) *Command {
+func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun, frozen, usePrerelease bool) *Command {
 	l, err := stencil.LoadLockfile("")
 	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		log.WithError(err).Warn("failed to load lockfile")
 	}
 
+	if usePrerelease {
+		log.Info("Using prerelease versions")
+		for i := range s.Modules {
+			s.Modules[i].Prerelease = usePrerelease
+		}
+	}
+
 	return &Command{
 		lock:           l,
 		manifest:       s,

internal/codegen/stencil.go

@@ -183,10 +183,23 @@ func (s *Stencil) getTemplates(ctx context.Context, log logrus.FieldLogger) ([]*
 			return nil, errors.Wrapf(err, "failed to read module filesystem %q", m.Name)
 		}
 
+		// Note: This error should never really fire since we already fetched the FS above
+		// that being said, we handle it here. Skip native extensions as they cannot have templates.
+		mf, err := m.Manifest(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if mf.Type != configuration.TemplateRepositoryTypeStd {
+			log.Debugf("Skipping template discovery for module %q, not a template module (type %s)", m.Name, mf.Type)
+			continue
+		}
+
 		log.Debugf("Discovering templates from module %q", m.Name)
 
 		// default to templates/, but if it's not present fallback to
 		// the root w/ a warning
+		// Note: This behaviour is deprecated and will be removed soon. Put templates
+		// into /templates instead.
 		if inf, err := fs.Stat("templates"); err != nil || !inf.IsDir() {
 			log.Warnf("Module %q has templates outside of templates/ directory, this is not recommended and is deprecated", m.Name)
 		} else {

internal/codegen/stencil_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/getoutreach/gobox/pkg/app"
 	"github.com/getoutreach/stencil/internal/modules"
+	"github.com/getoutreach/stencil/internal/modules/modulestest"
 	"github.com/getoutreach/stencil/pkg/configuration"
 	"github.com/getoutreach/stencil/pkg/stencil"
 	"github.com/go-git/go-billy/v5/memfs"
@@ -19,6 +20,11 @@ func TestBasicE2ERender(t *testing.T) {
 	fs := memfs.New()
 	ctx := context.Background()
 
+	// create stub manifest
+	f, _ := fs.Create("manifest.yaml")
+	f.Write([]byte("name: testing"))
+	f.Close()
+
 	// create a stub template
 	f, err := fs.Create("test-template.tpl")
 	assert.NilError(t, err, "failed to create stub template")
@@ -60,28 +66,22 @@ func TestBasicE2ERender(t *testing.T) {
 }
 
 func TestModuleHookRender(t *testing.T) {
-	m1fs := memfs.New()
-	m2fs := memfs.New()
 	ctx := context.Background()
 
-	// create a stub template
-	f, err := m1fs.Create("test-template.tpl")
-	assert.NilError(t, err, "failed to create stub template")
-	f.Write([]byte(`{{ file.Skip "virtual file" }}{{ stencil.AddToModuleHook "testing2" "coolthing" (list "a") }}`))
-	f.Close()
-
-	f, err = m2fs.Create("test-template.tpl")
-	assert.NilError(t, err, "failed to create stub template")
-	f.Write([]byte(`{{ index (stencil.GetModuleHook "coolthing") 0 }}`))
-	f.Close()
+	// create modules
+	m1, err := modulestest.NewModuleFromTemplates(nil, "testing1", "testdata/module-hook/m1.tpl")
+	if err != nil {
+		t.Errorf("failed to create module 1: %v", err)
+	}
+	m2, err := modulestest.NewModuleFromTemplates(nil, "testing2", "testdata/module-hook/m2.tpl")
+	if err != nil {
+		t.Errorf("failed to create module 2: %v", err)
+	}
 
 	st := NewStencil(&configuration.ServiceManifest{
 		Name:      "test",
 		Arguments: map[string]interface{}{},
-	}, []*modules.Module{
-		modules.NewWithFS(ctx, "testing1", m1fs),
-		modules.NewWithFS(ctx, "testing2", m2fs),
-	}, logrus.New())
+	}, []*modules.Module{m1, m2}, logrus.New())
 
 	tpls, err := st.Render(ctx, logrus.New())
 	assert.NilError(t, err, "expected Render() to not fail")

internal/codegen/testdata/module-hook/m1.tpl

@@ -0,0 +1,2 @@
+{{ file.Skip "virtual file" }}
+{{ stencil.AddToModuleHook "testing2" "coolthing" (list "a") }}

internal/codegen/testdata/module-hook/m2.tpl

@@ -0,0 +1 @@
+{{ index (stencil.GetModuleHook "coolthing") 0 }}

internal/modules/module.go

@@ -22,6 +22,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
+	gogithub "github.com/google/go-github/v43/github"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	giturls "github.com/whilp/git-urls"
@@ -52,37 +53,51 @@ type Module struct {
 
 // getLatestVersion returns the latest version of a git repository
 // name should be a valid go import path, like github.com/<org>/<repo>
-func getLatestVersion(ctx context.Context, name string) (string, error) {
-	paths := strings.Split(name, "/")
+func getLatestVersion(ctx context.Context, tr *configuration.TemplateRepository) (string, error) {
+	paths := strings.Split(tr.Name, "/")
 
 	// github.com, getoutreach, stencil-base
 	if len(paths) < 3 {
-		return "", fmt.Errorf("invalid module path %q, expected github.com/<org>/<repo>", name)
+		return "", fmt.Errorf("invalid module path %q, expected github.com/<org>/<repo>", tr.Name)
 	}
 
 	gh, err := github.NewClient(github.WithAllowUnauthenticated())
 	if err != nil {
 		return "", err
 	}
 
+	if tr.Prerelease { // Use the newest, first, release.
+		rels, _, err := gh.Repositories.ListReleases(ctx, paths[1], paths[2], &gogithub.ListOptions{
+			PerPage: 1,
+		})
+		if err != nil {
+			return "", errors.Wrapf(err, "failed to get releases for module %q", tr.Name)
+		}
+		if len(rels) != 1 {
+			return "", fmt.Errorf("failed to get latest release (returned >1 release) for module %q", tr.Name)
+		}
+		return rels[0].GetTagName(), nil
+	}
+
+	// Use GetLatestRelease() to ensure it's the latest _released_ version.
 	rel, _, err := gh.Repositories.GetLatestRelease(ctx, paths[1], paths[2])
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to find the latest release for module %q", name)
+		return "", errors.Wrapf(err, "failed to find the latest release for module %q", tr.Name)
 	}
 
 	return rel.GetTagName(), nil
 }
 
-// New creates a new module at a specific revision. If a revision is
-// not provided then the latest version is automatically used. A revision
-// must be a valid git ref of semantic version.
+// New creates a new module from a TemplateRepository. If no version is specified
+// then the latest released revision is used. If `prerelease` is set to true then
+// the latest revision is used, regardless of whether it is a released version or
+// not.
 //
-// If the uri is a file:// path, a version must be specified otherwise
-// it will be treated as if it has no version. If uri is not specified
-// it is default to HTTPS
-func New(ctx context.Context, name, uri, version string) (*Module, error) {
+// uri is the URI for the module. If it is an empty string https://+name is used
+// instead.
+func New(ctx context.Context, uri string, tr *configuration.TemplateRepository) (*Module, error) {
 	if uri == "" {
-		uri = "https://" + name
+		uri = "https://" + tr.Name
 	}
 
 	// check if a url based on if :// is in the uri, this is kinda hacky
@@ -91,28 +106,32 @@ func New(ctx context.Context, name, uri, version string) (*Module, error) {
 	if !strings.Contains(uri, "://") || strings.HasPrefix(uri, "file://") { // Assume it's a path.
 		osPath := strings.TrimPrefix(uri, "file://")
 		if _, err := os.Stat(osPath); err != nil {
-			return nil, errors.Wrapf(err, "failed to find module %s at path %q", name, osPath)
+			return nil, errors.Wrapf(err, "failed to find module %s at path %q", tr.Name, osPath)
 		}
 
 		// translate the path into a file:// URI
 		uri = "file://" + osPath
-		version = "local"
+		tr.Version = "local"
 	}
 
-	if version == "" {
+	if tr.Version == "" {
 		var err error
-		version, err = getLatestVersion(ctx, name)
+		tr.Version, err = getLatestVersion(ctx, tr)
 		if err != nil {
 			return nil, err
 		}
 	}
-	return &Module{template.New(name).Funcs(sprig.TxtFuncMap()), name, uri, version, nil}, nil
+	return &Module{template.New(tr.Name).Funcs(sprig.TxtFuncMap()), tr.Name, uri, tr.Version, nil}, nil
 }
 
 // NewWithFS creates a module with the specified file system. This is
 // generally only meant to be used in tests.
 func NewWithFS(ctx context.Context, name string, fs billy.Filesystem) *Module {
-	m, _ := New(ctx, name, "vfs://"+name, "vfs") //nolint:errcheck // Why: No errors
+	//nolint:errcheck // Why: No errors
+	m, _ := New(ctx, "vfs://"+name, &configuration.TemplateRepository{
+		Name:    name,
+		Version: "vfs",
+	})
 	m.fs = fs
 	return m
 }
@@ -137,12 +156,7 @@ func (m *Module) RegisterExtensions(ctx context.Context, log logrus.FieldLogger,
 		return nil
 	}
 
-	if m.Version != "" {
-		log.WithField("version", m.Version).
-			Warn("version was manually set on plugin, this is currently not supported, using latest")
-	}
-
-	return ext.RegisterExtension(ctx, m.URI, m.Name)
+	return ext.RegisterExtension(ctx, m.URI, m.Name, m.Version)
 }
 
 // Manifest downloads the module if not already downloaded and returns a parsed

internal/modules/module_test.go

@@ -15,7 +15,7 @@ import (
 
 func TestCanFetchModule(t *testing.T) {
 	ctx := context.Background()
-	m, err := modules.New(ctx, "github.com/getoutreach/stencil-base", "", "main")
+	m, err := modules.New(ctx, "", &configuration.TemplateRepository{Name: "github.com/getoutreach/stencil-base", Version: "main"})
 	assert.NilError(t, err, "failed to call New()")
 
 	manifest, err := m.Manifest(ctx)
@@ -30,7 +30,7 @@ func TestCanFetchModule(t *testing.T) {
 }
 
 func TestCanGetLatestModule(t *testing.T) {
-	_, err := modules.New(context.Background(), "github.com/getoutreach/stencil-base", "", "")
+	_, err := modules.New(context.Background(), "", &configuration.TemplateRepository{Name: "github.com/getoutreach/stencil-base"})
 	assert.NilError(t, err, "failed to call New()")
 }
 

internal/modules/modules.go

@@ -84,7 +84,10 @@ func getModulesForService(ctx context.Context, sm *configuration.ServiceManifest
 
 		// create a module struct for this module, this resolves the latest version if
 		// the version wasn't set.
-		m, err := New(ctx, d.Name, sm.Replacements[d.Name], d.Version)
+		m, err := New(ctx, sm.Replacements[d.Name], &configuration.TemplateRepository{
+			Name:    d.Name,
+			Version: d.Version,
+		})
 		if err != nil {
 			return errors.Wrapf(err, "failed to create dependency %q", d.Name)
 		}

internal/modules/modulestest/modulestest.go

@@ -41,7 +41,7 @@ func addTemplateToFS(fs billy.Filesystem, tpl string) error {
 
 // NewModuleFromTemplate creates a module with the provided template
 // being the only file in the module.
-func NewModuleFromTemplates(arguments map[string]configuration.Argument, templates ...string) (*modules.Module, error) {
+func NewModuleFromTemplates(arguments map[string]configuration.Argument, name string, templates ...string) (*modules.Module, error) {
 	fs := memfs.New()
 	for _, tpl := range templates {
 		if err := addTemplateToFS(fs, tpl); err != nil {
@@ -58,7 +58,7 @@ func NewModuleFromTemplates(arguments map[string]configuration.Argument, templat
 	// write a manifest file so that we can handle arguments
 	enc := yaml.NewEncoder(mf)
 	if err := enc.Encode(&configuration.TemplateRepositoryManifest{
-		Name:      "modulestest",
+		Name:      name,
 		Arguments: arguments,
 	}); err != nil {
 		return nil, errors.Wrap(err, "failed to encode generated module manifest")
@@ -68,5 +68,5 @@ func NewModuleFromTemplates(arguments map[string]configuration.Argument, templat
 	}
 
 	// create the module
-	return modules.NewWithFS(context.Background(), "modulestest", fs), nil
+	return modules.NewWithFS(context.Background(), name, fs), nil
 }

pkg/configuration/configuration.go

@@ -90,6 +90,9 @@ type TemplateRepository struct {
 	// Name is the name of this module. This should be a valid go import path
 	Name string `yaml:"name"`
 
+	// Prerelease is a boolean indicating whether or not to consider prerelease versions
+	Prerelease bool `yaml:"prerelease"`
+
 	// Deprecated: Use name instead
 	// URL is a full URL for a given module
 	URL string `yaml:"url,omitempty"`