diff --git a/cmd/stencil/stencil.go b/cmd/stencil/stencil.go index a698c8d5..18946d00 100644 --- a/cmd/stencil/stencil.go +++ b/cmd/stencil/stencil.go @@ -60,7 +60,7 @@ func main() { return errors.Wrap(err, "failed to parse service.yaml") } - cmd := stencil.NewCommand(log, serviceManifest, c.Bool("dry-run")) + cmd := stencil.NewCommand(log, serviceManifest, c.Bool("dry-run"), c.Bool("frozen-lockfile")) return errors.Wrap(cmd.Run(ctx), "run codegen") }, ///EndBlock(app) @@ -72,6 +72,10 @@ func main() { Aliases: []string{"dryrun"}, Usage: "Don't write files to disk", }, + &cli.BoolFlag{ + Name: "frozen-lockfile", + Usage: "Use versions from the lockfile instead of the latest", + }, ///EndBlock(flags) } app.Commands = []*cli.Command{ diff --git a/internal/cmd/stencil/stencil.go b/internal/cmd/stencil/stencil.go index 1deeb26c..bb6f1174 100644 --- a/internal/cmd/stencil/stencil.go +++ b/internal/cmd/stencil/stencil.go @@ -9,6 +9,7 @@ package stencil import ( "context" + "fmt" "os" "path/filepath" @@ -24,6 +25,9 @@ import ( // Command is a thin wrapper around the codegen package that // implements the "stencil" command. type Command struct { + // lock is the current stencil lockfile at command creation time + lock *stencil.Lockfile + // manifest is the service manifest that is being used // for this template render manifest *configuration.ServiceManifest @@ -33,19 +37,25 @@ type Command struct { // dryRun denotes if we should write files to disk or not dryRun bool + + // frozenLockfile denotes if we should use versions from the lockfile + // or not + frozenLockfile bool } // NewCommand creates a new stencil command -func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun bool) *Command { - _, err := stencil.LoadLockfile("") +func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun, frozen bool) *Command { + l, err := stencil.LoadLockfile("") if err != nil && !errors.Is(err, os.ErrNotExist) { log.WithError(err).Warn("failed to load lockfile") } return &Command{ - manifest: s, - log: log, - dryRun: dryRun, + lock: l, + manifest: s, + log: log, + dryRun: dryRun, + frozenLockfile: frozen, } } @@ -54,10 +64,16 @@ func NewCommand(log logrus.FieldLogger, s *configuration.ServiceManifest, dryRun // the templates. This step also does minimal post-processing of the dependencies // manifests func (c *Command) Run(ctx context.Context) error { - mods, err := modules.GetModulesForService(ctx, c.manifest) + c.log.Info("Fetching dependencies") + mods, err := modules.GetModulesForService(ctx, c.manifest, c.frozenLockfile, c.lock) if err != nil { return errors.Wrap(err, "failed to process modules list") } + + for _, m := range mods { + c.log.Infof(" -> %s %s", m.Name, m.Version) + } + st := codegen.NewStencil(c.manifest, mods) c.log.Info("Loading native extensions") @@ -71,15 +87,16 @@ func (c *Command) Run(ctx context.Context) error { return err } - // Below options mutate, so we shallow return - if c.dryRun { - return nil - } - if err := c.writeFiles(st, tpls); err != nil { return err } + // Can't dry run post run yet + if c.dryRun { + c.log.Info("Skipping post-run commands, dry-run") + return nil + } + return st.PostRun(ctx, c.log) } @@ -88,7 +105,10 @@ func (c *Command) writeFile(f *codegen.File) error { action := "Created" if f.Deleted { action = "Deleted" - os.Remove(f.Name()) + + if !c.dryRun { + os.Remove(f.Name()) + } } else if f.Skipped { action = "Skipped" } else if _, err := os.Stat(f.Name()); err == nil { @@ -96,16 +116,22 @@ func (c *Command) writeFile(f *codegen.File) error { } if action == "Created" || action == "Updated" { - if err := os.MkdirAll(filepath.Dir(f.Name()), 0o755); err != nil { - return errors.Wrapf(err, "failed to ensure directory for %q existed", f.Name()) - } + if !c.dryRun { + if err := os.MkdirAll(filepath.Dir(f.Name()), 0o755); err != nil { + return errors.Wrapf(err, "failed to ensure directory for %q existed", f.Name()) + } - if err := os.WriteFile(f.Name(), f.Bytes(), f.Mode()); err != nil { - return errors.Wrapf(err, "failed to create %q", f.Name()) + if err := os.WriteFile(f.Name(), f.Bytes(), f.Mode()); err != nil { + return errors.Wrapf(err, "failed to create %q", f.Name()) + } } } - c.log.Infof(" -> %s %s", action, f.Name()) + msg := fmt.Sprintf(" -> %s %s", action, f.Name()) + if c.dryRun { + msg += " (dry-run)" + } + c.log.Info(msg) return nil } @@ -121,6 +147,11 @@ func (c *Command) writeFiles(st *codegen.Stencil, tpls []*codegen.Template) erro } } + // Don't generate a lockfile in dry-run mode + if c.dryRun { + return nil + } + l := st.GenerateLockfile(tpls) f, err := os.Create(stencil.LockfileName) if err != nil { diff --git a/internal/modules/module.go b/internal/modules/module.go index ef289481..7ca2d644 100644 --- a/internal/modules/module.go +++ b/internal/modules/module.go @@ -83,7 +83,7 @@ func New(ctx context.Context, name, uri, version string) (*Module, error) { if uri == "" { uri = "https://" + name } else if strings.HasPrefix(uri, "file://") { - version = "local" + version = "local-" + strings.TrimPrefix(uri, "file://") } if version == "" { diff --git a/internal/modules/module_test.go b/internal/modules/module_test.go index 48ebf039..dd68ba39 100644 --- a/internal/modules/module_test.go +++ b/internal/modules/module_test.go @@ -10,6 +10,7 @@ import ( "github.com/getoutreach/stencil/internal/modules" "github.com/getoutreach/stencil/pkg/configuration" + "github.com/getoutreach/stencil/pkg/stencil" "gotest.tools/v3/assert" ) @@ -47,7 +48,7 @@ func TestReplacementLocalModule(t *testing.T) { }, } - mods, err := modules.GetModulesForService(context.Background(), sm) + mods, err := modules.GetModulesForService(context.Background(), sm, false, nil) assert.NilError(t, err, "expected GetModulesForService() to not error") assert.Equal(t, len(mods), 1, "expected exactly one module to be returned") assert.Equal(t, mods[0].URI, sm.Replacements["github.com/getoutreach/stencil-base"], @@ -64,8 +65,32 @@ func TestCanFetchDeprecatedModule(t *testing.T) { }, } - mods, err := modules.GetModulesForService(context.Background(), sm) + mods, err := modules.GetModulesForService(context.Background(), sm, false, nil) assert.NilError(t, err, "expected GetModulesForService() to not error") assert.Equal(t, len(mods), 1, "expected exactly one module to be returned") assert.Equal(t, mods[0].Name, "github.com/getoutreach/stencil-base") } + +func TestFrozenLockfile(t *testing.T) { + sm := &configuration.ServiceManifest{ + Name: "testing-service", + Modules: []*configuration.TemplateRepository{ + { + URL: "https://github.com/getoutreach/stencil-base", + }, + }, + } + + mods, err := modules.GetModulesForService(context.Background(), sm, true, &stencil.Lockfile{ + Modules: []*stencil.LockfileModuleEntry{ + { + Name: "github.com/getoutreach/stencil-base", + Version: "v0.0.7", + }, + }, + }) + assert.NilError(t, err, "expected GetModulesForService() to not error") + assert.Equal(t, len(mods), 1, "expected exactly one module to be returned") + assert.Equal(t, mods[0].Name, "github.com/getoutreach/stencil-base") + assert.Equal(t, mods[0].Version, "v0.0.7", "expected stencil-base version to be locked") +} diff --git a/internal/modules/modules.go b/internal/modules/modules.go index 0433e932..4681d63b 100644 --- a/internal/modules/modules.go +++ b/internal/modules/modules.go @@ -10,18 +10,59 @@ package modules import ( "context" + "fmt" "path" + "strings" "github.com/getoutreach/stencil/pkg/configuration" + "github.com/getoutreach/stencil/pkg/stencil" "github.com/pkg/errors" giturls "github.com/whilp/git-urls" ) +// IDEA(jaredallard): Remove m.URL support soon. + // GetModulesForService returns a list of modules that a given service manifest // depends on. They are not returned in the order of their import. -func GetModulesForService(ctx context.Context, m *configuration.ServiceManifest) ([]*Module, error) { - // We only support importing a single version of a module - // IDEA(jaredallard): Always use the latest? +func GetModulesForService(ctx context.Context, m *configuration.ServiceManifest, + frozen bool, l *stencil.Lockfile) ([]*Module, error) { + // If frozen, we iterate over the lockfile to set the versions + if frozen { + if l == nil { + return nil, fmt.Errorf("frozen lockfile requires a lockfile to exist") + } + + for _, m := range m.Modules { + // Convert m.URL -> m.Name + //nolint:staticcheck // Why: We're implementing compat here. + if m.URL != "" { + u, err := giturls.Parse(m.URL) //nolint:staticcheck // Why: We're implementing compat here. + if err != nil { + //nolint:staticcheck // Why: We're implementing compat here. + return nil, errors.Wrapf(err, "failed to parse deprecated url module syntax %q as a URL", m.URL) + } + m.Name = path.Join(u.Host, u.Path) + } + + for _, l := range l.Modules { + if m.Name == l.Name { + if strings.HasPrefix(l.URL, "file://") { + return nil, + fmt.Errorf("cannot use frozen lockfile for file dependency %q, re-add replacement or run without --frozen-lockfile", l.Name) + } + + m.Version = l.Version + break + } + } + if m.Version == "" { + return nil, fmt.Errorf("frozen lockfile, but no version found for module %q", m.Name) + } + } + } + + // create a map of modules, this is used to avoid downloading the same module twice + // as well as only ever including one version of a module modules := make(map[string]*Module) if err := getModulesForService(ctx, m, m.Modules, modules); err != nil { return nil, errors.Wrap(err, "failed to fetch modules") diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go index 6e233115..340a5d46 100644 --- a/pkg/configuration/configuration.go +++ b/pkg/configuration/configuration.go @@ -96,7 +96,7 @@ type TemplateRepository struct { // Version is a semantic version or branch of the template repository // that should be downloaded if not set then the latest version is used. - // Note: A single commit is currently not supported. + // Note: Setting this equates to pinning the versions, this is not recommended. Version string `yaml:"version"` } diff --git a/pkg/configuration/configuration_test.go b/pkg/configuration/configuration_test.go new file mode 100644 index 00000000..efb343bb --- /dev/null +++ b/pkg/configuration/configuration_test.go @@ -0,0 +1,41 @@ +// Copyright 2022 Outreach Corporation. All Rights Reserved. + +// Description: This file contains tests for the configuration pac + +package configuration_test + +import ( + "fmt" + + "github.com/getoutreach/stencil/pkg/configuration" +) + +func ExampleValidateName() { + // Normal name + success := configuration.ValidateName("test") + fmt.Println("success:", success) + + // Invalid name + success = configuration.ValidateName("test.1234") + fmt.Println("success:", success) + + // Output: + // success: true + // success: false +} + +func ExampleNewServiceManifest() { + sm, err := configuration.NewServiceManifest("testdata/service.yaml") + if err != nil { + // handle the error + fmt.Println("error:", err) + return + } + + fmt.Println(sm.Name) + fmt.Println(sm.Arguments) + + // Output: + // testing + // map[hello:world] +} diff --git a/pkg/configuration/testdata/service.yaml b/pkg/configuration/testdata/service.yaml new file mode 100644 index 00000000..d43ffb54 --- /dev/null +++ b/pkg/configuration/testdata/service.yaml @@ -0,0 +1,3 @@ +name: testing +arguments: + hello: world diff --git a/pkg/stencil/stencil_test.go b/pkg/stencil/stencil_test.go new file mode 100644 index 00000000..cd0ce544 --- /dev/null +++ b/pkg/stencil/stencil_test.go @@ -0,0 +1,27 @@ +// Copyright 2022 Outreach Corporation. All Rights Reserved. + +// Description: Contains tests for the stencil package + +package stencil_test + +import ( + "fmt" + "time" + + "github.com/getoutreach/stencil/pkg/stencil" +) + +func ExampleLoadLockfile() { + // Load the lockfile + l, err := stencil.LoadLockfile("testdata") + if err != nil { + // handle the error + fmt.Println(err) + return + } + + fmt.Println(l.Generated.UTC().Format(time.RFC3339Nano)) + + // Output: + // 2022-04-01T00:25:51.047307Z +} diff --git a/pkg/stencil/testdata/stencil.lock b/pkg/stencil/testdata/stencil.lock new file mode 100644 index 00000000..6aef9213 --- /dev/null +++ b/pkg/stencil/testdata/stencil.lock @@ -0,0 +1,4 @@ +version: v1.6.2 +generated: 2022-04-01T00:25:51.047307Z +modules: [] +files: [] diff --git a/service.yaml b/service.yaml index 9e9c2d11..ebe39e34 100644 --- a/service.yaml +++ b/service.yaml @@ -42,5 +42,3 @@ opsLevel: mainLink: "" custom: [] designDocumentLink: https://docs.google.com/document/d/1e2YaeyeHMj5HrKMrZU8fh7-KInQj023GfzDxCi2u09A/edit -replacements: - github.com/getoutreach/stencil-base: file://../stencil-base diff --git a/stencil.lock b/stencil.lock index d7c9d63d..a3b827dd 100644 --- a/stencil.lock +++ b/stencil.lock @@ -1,9 +1,9 @@ -version: v1.4.0-13-g555d45c -generated: 2022-03-17T05:37:47.707009Z +version: v1.6.2 +generated: 2022-04-01T00:25:51.047307Z modules: - name: github.com/getoutreach/stencil-base - url: file://../stencil-base - version: local + url: https://github.com/getoutreach/stencil-base + version: v0.0.7 files: - name: .github/CODEOWNERS template: .github/CODEOWNERS.tpl @@ -11,6 +11,12 @@ files: - name: .github/pull_request_template.md template: .github/pull_request_template.md.tpl module: github.com/getoutreach/stencil-base + - name: .gitignore + template: .gitignore.tpl + module: github.com/getoutreach/stencil-base + - name: .releaserc.yaml + template: .releaserc.yaml.tpl + module: github.com/getoutreach/stencil-base - name: CONTRIBUTING.md template: CONTRIBUTING.md.tpl module: github.com/getoutreach/stencil-base @@ -35,3 +41,18 @@ files: - name: documentation/runbook.md template: documentation/runbook.md.tpl module: github.com/getoutreach/stencil-base + - name: helpers + template: helpers.tpl + module: github.com/getoutreach/stencil-base + - name: package.json + template: package.json.tpl + module: github.com/getoutreach/stencil-base + - name: scripts/bootstrap-lib.sh + template: scripts/bootstrap-lib.sh.tpl + module: github.com/getoutreach/stencil-base + - name: scripts/devbase.sh + template: scripts/devbase.sh.tpl + module: github.com/getoutreach/stencil-base + - name: scripts/shell-wrapper.sh + template: scripts/shell-wrapper.sh.tpl + module: github.com/getoutreach/stencil-base