feat(stencil): add frozen-lockfile flag (#70)

* feat(stencil): add frozen-lockfile flag

* skip lockfile on dry-run, unpass lockfile to module resolver for now

* add test

cmd/stencil/stencil.go

@@ -60,7 +60,7 @@ func main() {
 				return errors.Wrap(err, "failed to parse service.yaml")
 			}
 
-			cmd := stencil.NewCommand(log, serviceManifest, c.Bool("dry-run"))
+			cmd := stencil.NewCommand(log, serviceManifest, c.Bool("dry-run"), c.Bool("frozen-lockfile"))
 			return errors.Wrap(cmd.Run(ctx), "run codegen")
 		},
 		///EndBlock(app)
@@ -72,6 +72,10 @@ func main() {
 			Aliases: []string{"dryrun"},
 			Usage:   "Don't write files to disk",
 		},
+		&cli.BoolFlag{
+			Name:  "frozen-lockfile",
+			Usage: "Use versions from the lockfile instead of the latest",
+		},
 		///EndBlock(flags)
 	}
 	app.Commands = []*cli.Command{

internal/cmd/stencil/stencil.go

@@ -9,6 +9,7 @@ package stencil
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -24,6 +25,9 @@ import (
 // Command is a thin wrapper around the codegen package that
 // implements the "stencil" command.
 type Command struct {
+	// lock is the current stencil lockfile at command creation time
+	lock *stencil.Lockfile
+
 	// manifest is the service manifest that is being used
 	// for this template render
 	manifest *configuration.ServiceManifest
@@ -33,19 +37,25 @@ type Command struct {
 
 	// dryRun denotes if we should write files to disk or not
 	dryRun bool
+
+	// frozenLockfile denotes if we should use versions from the lockfile
+	// or not
+	frozenLockfile bool
 }
 
 // NewCommand creates a new stencil command
-func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun bool) *Command {
-	_, err := stencil.LoadLockfile("")
+func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun, frozen bool) *Command {
+	l, err := stencil.LoadLockfile("")
 	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		log.WithError(err).Warn("failed to load lockfile")
 	}
 
 	return &Command{
-		manifest: s,
-		log:      log,
-		dryRun:   dryRun,
+		lock:           l,
+		manifest:       s,
+		log:            log,
+		dryRun:         dryRun,
+		frozenLockfile: frozen,
 	}
 }
 
@@ -54,10 +64,16 @@ func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun
 // the templates. This step also does minimal post-processing of the dependencies
 // manifests
 func (c *Command) Run(ctx context.Context) error {
-	mods, err := modules.GetModulesForService(ctx, c.manifest)
+	c.log.Info("Fetching dependencies")
+	mods, err := modules.GetModulesForService(ctx, c.manifest, c.frozenLockfile, c.lock)
 	if err != nil {
 		return errors.Wrap(err, "failed to process modules list")
 	}
+
+	for _, m := range mods {
+		c.log.Infof(" -> %s %s", m.Name, m.Version)
+	}
+
 	st := codegen.NewStencil(c.manifest, mods)
 
 	c.log.Info("Loading native extensions")
@@ -71,15 +87,16 @@ func (c *Command) Run(ctx context.Context) error {
 		return err
 	}
 
-	// Below options mutate, so we shallow return
-	if c.dryRun {
-		return nil
-	}
-
 	if err := c.writeFiles(st, tpls); err != nil {
 		return err
 	}
 
+	// Can't dry run post run yet
+	if c.dryRun {
+		c.log.Info("Skipping post-run commands, dry-run")
+		return nil
+	}
+
 	return st.PostRun(ctx, c.log)
 }
 
@@ -88,24 +105,33 @@ func (c *Command) writeFile(f *codegen.File) error {
 	action := "Created"
 	if f.Deleted {
 		action = "Deleted"
-		os.Remove(f.Name())
+
+		if !c.dryRun {
+			os.Remove(f.Name())
+		}
 	} else if f.Skipped {
 		action = "Skipped"
 	} else if _, err := os.Stat(f.Name()); err == nil {
 		action = "Updated"
 	}
 
 	if action == "Created" || action == "Updated" {
-		if err := os.MkdirAll(filepath.Dir(f.Name()), 0o755); err != nil {
-			return errors.Wrapf(err, "failed to ensure directory for %q existed", f.Name())
-		}
+		if !c.dryRun {
+			if err := os.MkdirAll(filepath.Dir(f.Name()), 0o755); err != nil {
+				return errors.Wrapf(err, "failed to ensure directory for %q existed", f.Name())
+			}
 
-		if err := os.WriteFile(f.Name(), f.Bytes(), f.Mode()); err != nil {
-			return errors.Wrapf(err, "failed to create %q", f.Name())
+			if err := os.WriteFile(f.Name(), f.Bytes(), f.Mode()); err != nil {
+				return errors.Wrapf(err, "failed to create %q", f.Name())
+			}
 		}
 	}
 
-	c.log.Infof("  -> %s %s", action, f.Name())
+	msg := fmt.Sprintf("  -> %s %s", action, f.Name())
+	if c.dryRun {
+		msg += " (dry-run)"
+	}
+	c.log.Info(msg)
 	return nil
 }
 
@@ -121,6 +147,11 @@ func (c *Command) writeFiles(st *codegen.Stencil, tpls []*codegen.Template) erro
 		}
 	}
 
+	// Don't generate a lockfile in dry-run mode
+	if c.dryRun {
+		return nil
+	}
+
 	l := st.GenerateLockfile(tpls)
 	f, err := os.Create(stencil.LockfileName)
 	if err != nil {

internal/modules/module.go

@@ -83,7 +83,7 @@ func New(ctx context.Context, name, uri, version string) (*Module, error) {
 	if uri == "" {
 		uri = "https://" + name
 	} else if strings.HasPrefix(uri, "file://") {
-		version = "local"
+		version = "local-" + strings.TrimPrefix(uri, "file://")
 	}
 
 	if version == "" {

internal/modules/module_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/getoutreach/stencil/internal/modules"
 	"github.com/getoutreach/stencil/pkg/configuration"
+	"github.com/getoutreach/stencil/pkg/stencil"
 	"gotest.tools/v3/assert"
 )
 
@@ -47,7 +48,7 @@ func TestReplacementLocalModule(t *testing.T) {
 		},
 	}
 
-	mods, err := modules.GetModulesForService(context.Background(), sm)
+	mods, err := modules.GetModulesForService(context.Background(), sm, false, nil)
 	assert.NilError(t, err, "expected GetModulesForService() to not error")
 	assert.Equal(t, len(mods), 1, "expected exactly one module to be returned")
 	assert.Equal(t, mods[0].URI, sm.Replacements["github.com/getoutreach/stencil-base"],
@@ -64,8 +65,32 @@ func TestCanFetchDeprecatedModule(t *testing.T) {
 		},
 	}
 
-	mods, err := modules.GetModulesForService(context.Background(), sm)
+	mods, err := modules.GetModulesForService(context.Background(), sm, false, nil)
 	assert.NilError(t, err, "expected GetModulesForService() to not error")
 	assert.Equal(t, len(mods), 1, "expected exactly one module to be returned")
 	assert.Equal(t, mods[0].Name, "github.com/getoutreach/stencil-base")
 }
+
+func TestFrozenLockfile(t *testing.T) {
+	sm := &configuration.ServiceManifest{
+		Name: "testing-service",
+		Modules: []*configuration.TemplateRepository{
+			{
+				URL: "https://github.com/getoutreach/stencil-base",
+			},
+		},
+	}
+
+	mods, err := modules.GetModulesForService(context.Background(), sm, true, &stencil.Lockfile{
+		Modules: []*stencil.LockfileModuleEntry{
+			{
+				Name:    "github.com/getoutreach/stencil-base",
+				Version: "v0.0.7",
+			},
+		},
+	})
+	assert.NilError(t, err, "expected GetModulesForService() to not error")
+	assert.Equal(t, len(mods), 1, "expected exactly one module to be returned")
+	assert.Equal(t, mods[0].Name, "github.com/getoutreach/stencil-base")
+	assert.Equal(t, mods[0].Version, "v0.0.7", "expected stencil-base version to be locked")
+}

internal/modules/modules.go

@@ -10,18 +10,59 @@ package modules
 
 import (
 	"context"
+	"fmt"
 	"path"
+	"strings"
 
 	"github.com/getoutreach/stencil/pkg/configuration"
+	"github.com/getoutreach/stencil/pkg/stencil"
 	"github.com/pkg/errors"
 	giturls "github.com/whilp/git-urls"
 )
 
+// IDEA(jaredallard): Remove m.URL support soon.
+
 // GetModulesForService returns a list of modules that a given service manifest
 // depends on. They are not returned in the order of their import.
-func GetModulesForService(ctx context.Context, m *configuration.ServiceManifest) ([]*Module, error) {
-	// We only support importing a single version of a module
-	// IDEA(jaredallard): Always use the latest?
+func GetModulesForService(ctx context.Context, m *configuration.ServiceManifest,
+	frozen bool, l *stencil.Lockfile) ([]*Module, error) {
+	// If frozen, we iterate over the lockfile to set the versions
+	if frozen {
+		if l == nil {
+			return nil, fmt.Errorf("frozen lockfile requires a lockfile to exist")
+		}
+
+		for _, m := range m.Modules {
+			// Convert m.URL -> m.Name
+			//nolint:staticcheck // Why: We're implementing compat here.
+			if m.URL != "" {
+				u, err := giturls.Parse(m.URL) //nolint:staticcheck // Why: We're implementing compat here.
+				if err != nil {
+					//nolint:staticcheck // Why: We're implementing compat here.
+					return nil, errors.Wrapf(err, "failed to parse deprecated url module syntax %q as a URL", m.URL)
+				}
+				m.Name = path.Join(u.Host, u.Path)
+			}
+
+			for _, l := range l.Modules {
+				if m.Name == l.Name {
+					if strings.HasPrefix(l.URL, "file://") {
+						return nil,
+							fmt.Errorf("cannot use frozen lockfile for file dependency %q, re-add replacement or run without --frozen-lockfile", l.Name)
+					}
+
+					m.Version = l.Version
+					break
+				}
+			}
+			if m.Version == "" {
+				return nil, fmt.Errorf("frozen lockfile, but no version found for module %q", m.Name)
+			}
+		}
+	}
+
+	// create a map of modules, this is used to avoid downloading the same module twice
+	// as well as only ever including one version of a module
 	modules := make(map[string]*Module)
 	if err := getModulesForService(ctx, m, m.Modules, modules); err != nil {
 		return nil, errors.Wrap(err, "failed to fetch modules")

pkg/configuration/configuration.go

@@ -96,7 +96,7 @@ type TemplateRepository struct {
 
 	// Version is a semantic version or branch of the template repository
 	// that should be downloaded if not set then the latest version is used.
-	// Note: A single commit is currently not supported.
+	// Note: Setting this equates to pinning the versions, this is not recommended.
 	Version string `yaml:"version"`
 }
 

pkg/configuration/configuration_test.go

@@ -0,0 +1,41 @@
+// Copyright 2022 Outreach Corporation. All Rights Reserved.
+
+// Description: This file contains tests for the configuration pac
+
+package configuration_test
+
+import (
+	"fmt"
+
+	"github.com/getoutreach/stencil/pkg/configuration"
+)
+
+func ExampleValidateName() {
+	// Normal name
+	success := configuration.ValidateName("test")
+	fmt.Println("success:", success)
+
+	// Invalid name
+	success = configuration.ValidateName("test.1234")
+	fmt.Println("success:", success)
+
+	// Output:
+	// success: true
+	// success: false
+}
+
+func ExampleNewServiceManifest() {
+	sm, err := configuration.NewServiceManifest("testdata/service.yaml")
+	if err != nil {
+		// handle the error
+		fmt.Println("error:", err)
+		return
+	}
+
+	fmt.Println(sm.Name)
+	fmt.Println(sm.Arguments)
+
+	// Output:
+	// testing
+	// map[hello:world]
+}

pkg/configuration/testdata/service.yaml

@@ -0,0 +1,3 @@
+name: testing
+arguments:
+  hello: world

pkg/stencil/stencil_test.go

@@ -0,0 +1,27 @@
+// Copyright 2022 Outreach Corporation. All Rights Reserved.
+
+// Description: Contains tests for the stencil package
+
+package stencil_test
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/getoutreach/stencil/pkg/stencil"
+)
+
+func ExampleLoadLockfile() {
+	// Load the lockfile
+	l, err := stencil.LoadLockfile("testdata")
+	if err != nil {
+		// handle the error
+		fmt.Println(err)
+		return
+	}
+
+	fmt.Println(l.Generated.UTC().Format(time.RFC3339Nano))
+
+	// Output:
+	// 2022-04-01T00:25:51.047307Z
+}

pkg/stencil/testdata/stencil.lock

@@ -0,0 +1,4 @@
+version: v1.6.2
+generated: 2022-04-01T00:25:51.047307Z
+modules: []
+files: []

service.yaml

@@ -42,5 +42,3 @@ opsLevel:
     mainLink: ""
     custom: []
   designDocumentLink: https://docs.google.com/document/d/1e2YaeyeHMj5HrKMrZU8fh7-KInQj023GfzDxCi2u09A/edit
-replacements:
-  github.com/getoutreach/stencil-base: file://../stencil-base