From f582ebc61c2d0c665e59623a27ad948d940a0547 Mon Sep 17 00:00:00 2001 From: Brandon Foley Date: Thu, 3 Oct 2024 13:30:19 -0400 Subject: [PATCH 1/6] Adding Generic Template Handler --- pkg/config/draftconfig.go | 47 ++++ pkg/config/draftconfig_template_test.go | 4 +- pkg/deployments/deployments.go | 111 -------- pkg/deployments/deployments_test.go | 252 ------------------ pkg/handlers/template.go | 146 ++++++++++ pkg/handlers/template_utils.go | 76 ++++++ pkg/handlers/template_utils_test.go | 20 ++ pkg/workflows/manifests/deployment.yaml | 22 ++ .../overlays/production/deployment.yaml | 12 + template/README.md | 43 +++ template/dockerfiles/gradlew/draft.yaml | 2 +- 11 files changed, 370 insertions(+), 365 deletions(-) delete mode 100644 pkg/deployments/deployments.go delete mode 100644 pkg/deployments/deployments_test.go create mode 100644 pkg/handlers/template.go create mode 100644 pkg/handlers/template_utils.go create mode 100644 pkg/handlers/template_utils_test.go create mode 100644 pkg/workflows/manifests/deployment.yaml create mode 100644 pkg/workflows/overlays/production/deployment.yaml create mode 100644 template/README.md diff --git a/pkg/config/draftconfig.go b/pkg/config/draftconfig.go index fc7c57e3..63d13e85 100644 --- a/pkg/config/draftconfig.go +++ b/pkg/config/draftconfig.go @@ -7,6 +7,8 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/blang/semver/v4" ) const draftConfigFile = "draft.yaml" @@ -125,6 +127,51 @@ func (d *DraftConfig) ApplyDefaultVariables() error { return nil } +func (d *DraftConfig) ApplyDefaultVariablesForVersion(version string) error { + v, err := semver.Parse(version) + if err != nil { + return fmt.Errorf("invalid version: %w", err) + } + + for _, variable := range d.Variables { + if variable.Value == "" { + expectedRange, err := semver.ParseRange(variable.Versions) + if err != nil { + return fmt.Errorf("invalid variable versions: %w", err) + } + + if !expectedRange(v) { + log.Infof("Variable %s versions %s is outside input version %s, skipping", variable.Name, variable.Versions, version) + continue + } + + if variable.Default.ReferenceVar != "" { + referenceVar, err := d.GetVariable(variable.Default.ReferenceVar) + if err != nil { + return fmt.Errorf("apply default variables: %w", err) + } + defaultVal, err := d.recurseReferenceVars(referenceVar, referenceVar, true) + if err != nil { + return fmt.Errorf("apply default variables: %w", err) + } + log.Infof("Variable %s defaulting to value %s", variable.Name, defaultVal) + variable.Value = defaultVal + } + + if variable.Value == "" { + if variable.Default.Value != "" { + log.Infof("Variable %s defaulting to value %s", variable.Name, variable.Default.Value) + variable.Value = variable.Default.Value + } else { + return errors.New("variable " + variable.Name + " has no default value") + } + } + } + } + + return nil +} + // recurseReferenceVars recursively checks each variable's ReferenceVar if it doesn't have a custom input. If there's no more ReferenceVars, it will return the default value of the last ReferenceVar. func (d *DraftConfig) recurseReferenceVars(referenceVar *BuilderVar, variableCheck *BuilderVar, isFirst bool) (string, error) { if !isFirst && referenceVar.Name == variableCheck.Name { diff --git a/pkg/config/draftconfig_template_test.go b/pkg/config/draftconfig_template_test.go index 7b937100..89a2d504 100644 --- a/pkg/config/draftconfig_template_test.go +++ b/pkg/config/draftconfig_template_test.go @@ -88,7 +88,7 @@ func loadTemplatesWithValidation() error { return fmt.Errorf("template %s has no template name", path) } - if _, ok := allTemplates[currTemplate.TemplateName]; ok { + if _, ok := allTemplates[strings.ToLower(currTemplate.TemplateName)]; ok { return fmt.Errorf("template %s has a duplicate template name", path) } @@ -109,6 +109,8 @@ func loadTemplatesWithValidation() error { return fmt.Errorf("template %s has an invalid variable kind: %s", path, variable.Kind) } } + + allTemplates[strings.ToLower(currTemplate.TemplateName)] = currTemplate return nil }) } diff --git a/pkg/deployments/deployments.go b/pkg/deployments/deployments.go deleted file mode 100644 index 5275478c..00000000 --- a/pkg/deployments/deployments.go +++ /dev/null @@ -1,111 +0,0 @@ -package deployments - -import ( - "embed" - "fmt" - "io/fs" - "path" - - "golang.org/x/exp/maps" - "gopkg.in/yaml.v3" - - log "github.com/sirupsen/logrus" - - "github.com/Azure/draft/pkg/config" - "github.com/Azure/draft/pkg/embedutils" - "github.com/Azure/draft/pkg/osutil" - "github.com/Azure/draft/pkg/templatewriter" -) - -const ( - parentDirName = "deployments" - configFileName = "draft.yaml" -) - -type Deployments struct { - deploys map[string]fs.DirEntry - configs map[string]*config.DraftConfig - dest string - deploymentTemplates fs.FS -} - -// DeployTypes returns a slice of the supported deployment types -func (d *Deployments) DeployTypes() []string { - names := maps.Keys(d.deploys) - return names -} - -func (d *Deployments) CopyDeploymentFiles(deployType string, deployConfig *config.DraftConfig, templateWriter templatewriter.TemplateWriter) error { - val, ok := d.deploys[deployType] - if !ok { - return fmt.Errorf("deployment type: %s is not currently supported", deployType) - } - - srcDir := path.Join(parentDirName, val.Name()) - - if err := deployConfig.ApplyDefaultVariables(); err != nil { - return fmt.Errorf("create deployment files for deployment type: %w", err) - } - - if err := osutil.CopyDirWithTemplates(d.deploymentTemplates, srcDir, d.dest, deployConfig, templateWriter); err != nil { - return err - } - - return nil -} - -func (d *Deployments) loadConfig(deployType string) (*config.DraftConfig, error) { - val, ok := d.deploys[deployType] - if !ok { - return nil, fmt.Errorf("deployment type %s unsupported", deployType) - } - - configPath := path.Join(parentDirName, val.Name(), configFileName) - configBytes, err := fs.ReadFile(d.deploymentTemplates, configPath) - if err != nil { - return nil, err - } - - var draftConfig config.DraftConfig - if err = yaml.Unmarshal(configBytes, &draftConfig); err != nil { - return nil, err - } - - return &draftConfig, nil -} - -func (d *Deployments) GetConfig(deployType string) (*config.DraftConfig, error) { - val, ok := d.configs[deployType] - if !ok { - return nil, fmt.Errorf("deployment type: %s is not currently supported", deployType) - } - return val, nil -} - -func (d *Deployments) PopulateConfigs() { - for deployType := range d.deploys { - draftConfig, err := d.loadConfig(deployType) - if err != nil { - log.Debugf("no draftConfig found for deployment type %s", deployType) - draftConfig = &config.DraftConfig{} - } - d.configs[deployType] = draftConfig - } -} - -func CreateDeploymentsFromEmbedFS(deploymentTemplates embed.FS, dest string) *Deployments { - deployMap, err := embedutils.EmbedFStoMap(deploymentTemplates, parentDirName) - if err != nil { - log.Fatal(err) - } - - d := &Deployments{ - deploys: deployMap, - dest: dest, - configs: make(map[string]*config.DraftConfig), - deploymentTemplates: deploymentTemplates, - } - d.PopulateConfigs() - - return d -} diff --git a/pkg/deployments/deployments_test.go b/pkg/deployments/deployments_test.go deleted file mode 100644 index 3da9259f..00000000 --- a/pkg/deployments/deployments_test.go +++ /dev/null @@ -1,252 +0,0 @@ -package deployments - -import ( - "embed" - "fmt" - "github.com/Azure/draft/pkg/config" - "github.com/Azure/draft/pkg/embedutils" - "github.com/Azure/draft/pkg/fixtures" - "github.com/Azure/draft/pkg/templatewriter/writers" - "io" - "io/fs" - "os" - "testing" - "testing/fstest" - - "github.com/Azure/draft/template" - "github.com/stretchr/testify/assert" -) - -var testFS embed.FS - -func TestCreateDeployments(t *testing.T) { - dest := "." - templateWriter := &writers.LocalFSWriter{} - draftConfig := &config.DraftConfig{ - Variables: []*config.BuilderVar{ - {Name: "APPNAME", Value: "testapp"}, - {Name: "NAMESPACE", Value: "default"}, - {Name: "PORT", Value: "80"}, - {Name: "IMAGENAME", Value: "testimage"}, - {Name: "IMAGETAG", Value: "latest"}, - {Name: "GENERATORLABEL", Value: "draft"}, - {Name: "SERVICEPORT", Value: "80"}, - }, - } - - tests := []struct { - name string - deployType string - shouldError bool - tempDirPath string - tempFileName string - tempPath string - fixturePath string - cleanUp func() - }{ - { - name: "helm", - deployType: "helm", - shouldError: false, - tempDirPath: "charts/templates", - tempFileName: "charts/templates/deployment.yaml", - tempPath: "../../test/templates/helm/charts/templates/deployment.yaml", - fixturePath: "../fixtures/deployments/charts/templates/deployment.yaml", - cleanUp: func() { - os.Remove(".charts") - }, - }, - { - name: "unsupported", - deployType: "unsupported", - shouldError: true, - tempDirPath: "test/templates/unsupported", - tempFileName: "test/templates/unsupported/deployment.yaml", - tempPath: "test/templates/unsupported/deployment.yaml", - cleanUp: func() { - os.Remove("deployments") - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fmt.Println("Creating temp file:", tt.tempFileName) - err := createTempDeploymentFile(tt.tempDirPath, tt.tempFileName, tt.tempPath) - assert.Nil(t, err) - - deployments := CreateDeploymentsFromEmbedFS(template.Deployments, dest) - err = deployments.CopyDeploymentFiles(tt.deployType, draftConfig, templateWriter) - if tt.shouldError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - - generatedContent, err := os.ReadFile(tt.tempFileName) - assert.Nil(t, err) - - if _, err := os.Stat(tt.fixturePath); os.IsNotExist(err) { - t.Errorf("Fixture file does not exist at path: %s", tt.fixturePath) - } - - err = fixtures.ValidateContentAgainstFixture(generatedContent, tt.fixturePath) - assert.Nil(t, err) - } - - tt.cleanUp() - }) - } -} - -func TestLoadConfig(t *testing.T) { - fakeFS, err := createMockDeploymentTemplatesFS() - assert.Nil(t, err) - - d, err := createMockDeployments("deployments", fakeFS) - assert.Nil(t, err) - - cases := []loadConfTestCase{ - {"helm", true}, - {"unsupported", false}, - } - - for _, c := range cases { - if c.isNil { - _, err = d.loadConfig(c.deployType) - assert.Nil(t, err) - } else { - _, err = d.loadConfig(c.deployType) - assert.NotNil(t, err) - } - } -} - -func TestPopulateConfigs(t *testing.T) { - fakeFS, err := createMockDeploymentTemplatesFS() - assert.Nil(t, err) - - d, err := createMockDeployments("deployments", fakeFS) - assert.Nil(t, err) - - d.PopulateConfigs() - assert.Equal(t, 3, len(d.configs)) - - d, err = createTestDeploymentEmbed("deployments") - assert.Nil(t, err) - - d.PopulateConfigs() - assert.Equal(t, 3, len(d.configs)) -} - -type loadConfTestCase struct { - deployType string - isNil bool -} - -func createTempDeploymentFile(dirPath, fileName, path string) error { - err := os.MkdirAll(dirPath, 0755) - if err != nil { - return err - } - file, err := os.Create(fileName) - if err != nil { - return err - } - fmt.Printf("file %v\n", file) - defer file.Close() - - var source *os.File - source, err = os.Open(path) - if err != nil { - return err - } - fmt.Printf("source %v\n", source) - defer source.Close() - - _, err = io.Copy(file, source) - if err != nil { - return err - } - return nil -} - -func createMockDeploymentTemplatesFS() (fs.FS, error) { - rootPath := "deplyments/" - embedFiles, err := embedutils.EmbedFStoMapWithFiles(template.Deployments, "deployments") - if err != nil { - return nil, fmt.Errorf("failed to readDir: %w in embeded files", err) - } - - mockFS := fstest.MapFS{} - - for path, file := range embedFiles { - if file.IsDir() { - mockFS[path] = &fstest.MapFile{Mode: fs.ModeDir} - } else { - bytes, err := template.Deployments.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failes to read file: %w", err) - } - mockFS[path] = &fstest.MapFile{Data: bytes} - } - } - - mockFS[rootPath+"emptyDir"] = &fstest.MapFile{Mode: fs.ModeDir} - mockFS[rootPath+"corrupted"] = &fstest.MapFile{Mode: fs.ModeDir} - mockFS[rootPath+"corrupted/draft.yaml"] = &fstest.MapFile{Data: []byte("fake yaml data")} - - return mockFS, nil -} - -func createMockDeployments(dirPath string, mockDeployments fs.FS) (*Deployments, error) { - dest := "." - - deployMap, err := fsToMap(mockDeployments, dirPath) - if err != nil { - return nil, fmt.Errorf("failed fsToMap: %w", err) - } - - d := &Deployments{ - deploys: deployMap, - dest: dest, - configs: make(map[string]*config.DraftConfig), - deploymentTemplates: mockDeployments, - } - - return d, nil -} - -func createTestDeploymentEmbed(dirPath string) (*Deployments, error) { - dest := "." - - deployMap, err := embedutils.EmbedFStoMap(template.Deployments, "deployments") - if err != nil { - return nil, fmt.Errorf("failed to create deployMap: %w", err) - } - - d := &Deployments{ - deploys: deployMap, - dest: dest, - configs: make(map[string]*config.DraftConfig), - deploymentTemplates: template.Deployments, - } - - return d, nil -} - -func fsToMap(fsFS fs.FS, path string) (map[string]fs.DirEntry, error) { - files, err := fs.ReadDir(fsFS, path) - if err != nil { - return nil, fmt.Errorf("failed to ReadDir: %w", err) - } - - mapping := make(map[string]fs.DirEntry) - - for _, f := range files { - if f.IsDir() { - mapping[f.Name()] = f - } - } - - return mapping, nil -} diff --git a/pkg/handlers/template.go b/pkg/handlers/template.go new file mode 100644 index 00000000..7f20e1b9 --- /dev/null +++ b/pkg/handlers/template.go @@ -0,0 +1,146 @@ +package handlers + +import ( + "bytes" + "fmt" + "io/fs" + "log" + "path/filepath" + "strings" + tmpl "text/template" + + "github.com/Azure/draft/pkg/config" + "github.com/Azure/draft/pkg/templatewriter" +) + +type Template struct { + Config *config.DraftConfig + + templateFiles fs.FS + templateWriter templatewriter.TemplateWriter + src string + dest string + version string +} + +// GetTemplate returns a template by name, version, and destination +func GetTemplate(name, version, dest string, templateWriter templatewriter.TemplateWriter) (*Template, error) { + template, ok := templateConfigs[strings.ToLower(name)] + if !ok { + return nil, fmt.Errorf("template not found: %s", name) + } + + if version == "" { + version = template.Config.DefaultVersion + log.Println("version not provided, using default version: ", version) + } + + if !IsValidVersion(template.Config.Versions, version) { + return nil, fmt.Errorf("invalid version: %s", version) + } + + if dest == "" { + dest = "." + log.Println("destination not provided, using current directory") + } + + if _, err := filepath.Abs(dest); err != nil { + return nil, fmt.Errorf("invalid destination: %s", dest) + } + + template.dest = dest + template.version = version + template.templateWriter = templateWriter + + return template, nil +} + +func (t *Template) Generate() error { + if err := t.validate(); err != nil { + log.Printf("template validation failed: %s", err.Error()) + return err + } + + if err := t.Config.ApplyDefaultVariablesForVersion(t.version); err != nil { + return fmt.Errorf("create workflow files: %w", err) + } + + if err := generateTemplate(t); err != nil { + return err + } + return generateTemplate(t) +} + +func (t *Template) validate() error { + if t == nil { + return fmt.Errorf("template is nil") + } + + if t.Config == nil { + return fmt.Errorf("template draft config is nil") + } + + if t.src == "" { + return fmt.Errorf("template source is empty") + } + + if t.dest == "" { + return fmt.Errorf("template destination is empty") + } + + if t.templateFiles == nil { + return fmt.Errorf("template files is nil") + } + + if t.version == "" { + return fmt.Errorf("template version is empty") + } + + return nil +} + +func generateTemplate(template *Template) error { + err := fs.WalkDir(template.templateFiles, template.src, func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + + if strings.EqualFold(d.Name(), "draft.yaml") { + return nil + } + + if err := writeTemplate(template, path); err != nil { + return err + } + + return nil + }) + + return err +} + +func writeTemplate(draftTemplate *Template, inputFile string) error { + file, err := fs.ReadFile(draftTemplate.templateFiles, inputFile) + if err != nil { + return err + } + + // Parse the template file, missingkey=error ensures an error will be returned if any variable is missing during template execution. + tmpl, err := tmpl.New("template").Option("missingkey=error").Parse(string(file)) + if err != nil { + return err + } + + // Execute the template with variableMap + var buf bytes.Buffer + err = tmpl.Execute(&buf, draftTemplate) + if err != nil { + return err + } + + if err = draftTemplate.templateWriter.WriteFile(fmt.Sprintf("%s/%s", draftTemplate.dest, filepath.Base(inputFile)), buf.Bytes()); err != nil { + return err + } + + return nil +} diff --git a/pkg/handlers/template_utils.go b/pkg/handlers/template_utils.go new file mode 100644 index 00000000..9c073460 --- /dev/null +++ b/pkg/handlers/template_utils.go @@ -0,0 +1,76 @@ +package handlers + +import ( + "fmt" + "io/fs" + "log" + "path/filepath" + "strings" + + "github.com/Azure/draft/pkg/config" + "github.com/Azure/draft/template" + "github.com/blang/semver/v4" +) + +var templateConfigs map[string]*Template + +func init() { + if err := loadTemplates(); err != nil { + log.Fatalf("failed to init templates: %s", err.Error()) + } +} + +// GetTemplates returns all templates +func GetTemplates() map[string]*Template { + return templateConfigs +} + +func loadTemplates() error { + templateConfigs = make(map[string]*Template) + return fs.WalkDir(template.Templates, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + if !strings.EqualFold(d.Name(), "draft.yaml") { + return nil + } + + draftConfig, err := config.NewConfigFromFS(template.Templates, path) + if err != nil { + return err + } + + if _, ok := templateConfigs[strings.ToLower(draftConfig.TemplateName)]; ok { + return fmt.Errorf("duplicate template name: %s", draftConfig.TemplateName) + } + + newTemplate := &Template{ + Config: draftConfig, + src: filepath.Dir(path), + templateFiles: template.Templates, + } + + templateConfigs[strings.ToLower(draftConfig.TemplateName)] = newTemplate + return nil + }) +} + +// IsValidVersion checks if a version is valid for a given version range +func IsValidVersion(versionRange, version string) bool { + v, err := semver.Parse(version) + if err != nil { + return false + } + + expectedRange, err := semver.ParseRange(versionRange) + if err != nil { + return false + } + + return expectedRange(v) +} diff --git a/pkg/handlers/template_utils_test.go b/pkg/handlers/template_utils_test.go new file mode 100644 index 00000000..25da381e --- /dev/null +++ b/pkg/handlers/template_utils_test.go @@ -0,0 +1,20 @@ +package handlers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetTemplate(t *testing.T) { + loadedTemplates := GetTemplates() + assert.Positive(t, len(loadedTemplates)) +} + +func TestLoadTemplates(t *testing.T) { + templateConfigs = nil + err := loadTemplates() + assert.Nil(t, err) + loadedTemplates := GetTemplates() + assert.Positive(t, len(loadedTemplates)) +} diff --git a/pkg/workflows/manifests/deployment.yaml b/pkg/workflows/manifests/deployment.yaml new file mode 100644 index 00000000..d339c904 --- /dev/null +++ b/pkg/workflows/manifests/deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test + labels: + app: test + kubernetes.azure.com/generator: draft +spec: + replicas: 1 + selector: + matchLabels: + app: test + template: + metadata: + labels: + app: test + spec: + containers: + - name: test + image: test + ports: + - containerPort: 8000 \ No newline at end of file diff --git a/pkg/workflows/overlays/production/deployment.yaml b/pkg/workflows/overlays/production/deployment.yaml new file mode 100644 index 00000000..11621fba --- /dev/null +++ b/pkg/workflows/overlays/production/deployment.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test + labels: + kubernetes.azure.com/generator: draft +spec: + template: + spec: + containers: + - name: test + image: acr.test:latest \ No newline at end of file diff --git a/template/README.md b/template/README.md new file mode 100644 index 00000000..d2390baf --- /dev/null +++ b/template/README.md @@ -0,0 +1,43 @@ +# TEMPLATE DEFINITION DOCS + +## Definitions + +All templates are defined within the `./template` directory with a cluster of go template files accompanied by a `draft.yaml` file. + +### draft.yaml + +The `draft.yaml` file contains the metadata needed to define a Template in Draft. The structure of the `draft.yaml` is as follows: + +- `templateName` - The name of the template +- `type` - The type of template +- `description` - Description of template contents/functionality +- `versions` - the range/list of version definitions for this template +- `defaultVersions` - If no version is passed to a template this will be used +- `parameters` - a struct containing information on each parameter to the template + - `name` - the parameter name associated to the gotemplate variable + - `description` - description of what the parameter is used for + - `type` - defines the type of the parameter + - `kind` - defines the kind of parameter, useful for prompting and validation within portal/cli/vsce + - `required` - defines if the parameter is required for the template + - `default` - struct containing information on specific parameters default value + - `value` - the parameters default value + - `referenceVar` - the variable to reference if one is not provided + - `versions` - the versions this item is used for + +For the `type` parameters at the template level we currently have 4 definitions: +- `deployment` - the base k8s deployment + service + namespace +- `dockerfile` - representing a dockerfile for a specific language +- `workflow` - representing a GitHub Action, ADO Pipeline, or similar +- `manifest` - a generic k8s manifest. Think PDB, Ingress, HPA that can be added to an existing `deployment` + +For the `type` parameter at the variable level, this is in line with structured types: `int`, `float`, `string`, `bool`, `object`. + +For the `kind` parameter, this will be used for validation and transformation logic on the input. As an example, `azureResourceGroup` and `azureResourceName` can be validated as defined. + +### Validation + +Within the [draft config teamplate tests](../pkg/config/draftconfig_template_test.go) there is validation logic to make sure all `draft.yaml` definitions adhere to: +- Unique `templateName`'s +- Valid Template `type`'s +- Valid parameter `type`'s +- Valid parameter `kind`'s \ No newline at end of file diff --git a/template/dockerfiles/gradlew/draft.yaml b/template/dockerfiles/gradlew/draft.yaml index 182c5117..22e07882 100644 --- a/template/dockerfiles/gradlew/draft.yaml +++ b/template/dockerfiles/gradlew/draft.yaml @@ -1,6 +1,6 @@ language: gradle displayName: Gradle -templateName: "dockerfile-gradle" +templateName: "dockerfile-gradlew" description: "This template is used to create a Dockerfile for a Gradle application" type: "dockerfile" variables: From 62d4b2ae2049a7704392fb9db7b52db8234ab721 Mon Sep 17 00:00:00 2001 From: Brandon Foley Date: Thu, 3 Oct 2024 14:28:52 -0400 Subject: [PATCH 2/6] add back deployments dir --- pkg/deployments/deployments.go | 111 ++++++++++++ pkg/deployments/deployments_test.go | 252 ++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 pkg/deployments/deployments.go create mode 100644 pkg/deployments/deployments_test.go diff --git a/pkg/deployments/deployments.go b/pkg/deployments/deployments.go new file mode 100644 index 00000000..5275478c --- /dev/null +++ b/pkg/deployments/deployments.go @@ -0,0 +1,111 @@ +package deployments + +import ( + "embed" + "fmt" + "io/fs" + "path" + + "golang.org/x/exp/maps" + "gopkg.in/yaml.v3" + + log "github.com/sirupsen/logrus" + + "github.com/Azure/draft/pkg/config" + "github.com/Azure/draft/pkg/embedutils" + "github.com/Azure/draft/pkg/osutil" + "github.com/Azure/draft/pkg/templatewriter" +) + +const ( + parentDirName = "deployments" + configFileName = "draft.yaml" +) + +type Deployments struct { + deploys map[string]fs.DirEntry + configs map[string]*config.DraftConfig + dest string + deploymentTemplates fs.FS +} + +// DeployTypes returns a slice of the supported deployment types +func (d *Deployments) DeployTypes() []string { + names := maps.Keys(d.deploys) + return names +} + +func (d *Deployments) CopyDeploymentFiles(deployType string, deployConfig *config.DraftConfig, templateWriter templatewriter.TemplateWriter) error { + val, ok := d.deploys[deployType] + if !ok { + return fmt.Errorf("deployment type: %s is not currently supported", deployType) + } + + srcDir := path.Join(parentDirName, val.Name()) + + if err := deployConfig.ApplyDefaultVariables(); err != nil { + return fmt.Errorf("create deployment files for deployment type: %w", err) + } + + if err := osutil.CopyDirWithTemplates(d.deploymentTemplates, srcDir, d.dest, deployConfig, templateWriter); err != nil { + return err + } + + return nil +} + +func (d *Deployments) loadConfig(deployType string) (*config.DraftConfig, error) { + val, ok := d.deploys[deployType] + if !ok { + return nil, fmt.Errorf("deployment type %s unsupported", deployType) + } + + configPath := path.Join(parentDirName, val.Name(), configFileName) + configBytes, err := fs.ReadFile(d.deploymentTemplates, configPath) + if err != nil { + return nil, err + } + + var draftConfig config.DraftConfig + if err = yaml.Unmarshal(configBytes, &draftConfig); err != nil { + return nil, err + } + + return &draftConfig, nil +} + +func (d *Deployments) GetConfig(deployType string) (*config.DraftConfig, error) { + val, ok := d.configs[deployType] + if !ok { + return nil, fmt.Errorf("deployment type: %s is not currently supported", deployType) + } + return val, nil +} + +func (d *Deployments) PopulateConfigs() { + for deployType := range d.deploys { + draftConfig, err := d.loadConfig(deployType) + if err != nil { + log.Debugf("no draftConfig found for deployment type %s", deployType) + draftConfig = &config.DraftConfig{} + } + d.configs[deployType] = draftConfig + } +} + +func CreateDeploymentsFromEmbedFS(deploymentTemplates embed.FS, dest string) *Deployments { + deployMap, err := embedutils.EmbedFStoMap(deploymentTemplates, parentDirName) + if err != nil { + log.Fatal(err) + } + + d := &Deployments{ + deploys: deployMap, + dest: dest, + configs: make(map[string]*config.DraftConfig), + deploymentTemplates: deploymentTemplates, + } + d.PopulateConfigs() + + return d +} diff --git a/pkg/deployments/deployments_test.go b/pkg/deployments/deployments_test.go new file mode 100644 index 00000000..3da9259f --- /dev/null +++ b/pkg/deployments/deployments_test.go @@ -0,0 +1,252 @@ +package deployments + +import ( + "embed" + "fmt" + "github.com/Azure/draft/pkg/config" + "github.com/Azure/draft/pkg/embedutils" + "github.com/Azure/draft/pkg/fixtures" + "github.com/Azure/draft/pkg/templatewriter/writers" + "io" + "io/fs" + "os" + "testing" + "testing/fstest" + + "github.com/Azure/draft/template" + "github.com/stretchr/testify/assert" +) + +var testFS embed.FS + +func TestCreateDeployments(t *testing.T) { + dest := "." + templateWriter := &writers.LocalFSWriter{} + draftConfig := &config.DraftConfig{ + Variables: []*config.BuilderVar{ + {Name: "APPNAME", Value: "testapp"}, + {Name: "NAMESPACE", Value: "default"}, + {Name: "PORT", Value: "80"}, + {Name: "IMAGENAME", Value: "testimage"}, + {Name: "IMAGETAG", Value: "latest"}, + {Name: "GENERATORLABEL", Value: "draft"}, + {Name: "SERVICEPORT", Value: "80"}, + }, + } + + tests := []struct { + name string + deployType string + shouldError bool + tempDirPath string + tempFileName string + tempPath string + fixturePath string + cleanUp func() + }{ + { + name: "helm", + deployType: "helm", + shouldError: false, + tempDirPath: "charts/templates", + tempFileName: "charts/templates/deployment.yaml", + tempPath: "../../test/templates/helm/charts/templates/deployment.yaml", + fixturePath: "../fixtures/deployments/charts/templates/deployment.yaml", + cleanUp: func() { + os.Remove(".charts") + }, + }, + { + name: "unsupported", + deployType: "unsupported", + shouldError: true, + tempDirPath: "test/templates/unsupported", + tempFileName: "test/templates/unsupported/deployment.yaml", + tempPath: "test/templates/unsupported/deployment.yaml", + cleanUp: func() { + os.Remove("deployments") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Println("Creating temp file:", tt.tempFileName) + err := createTempDeploymentFile(tt.tempDirPath, tt.tempFileName, tt.tempPath) + assert.Nil(t, err) + + deployments := CreateDeploymentsFromEmbedFS(template.Deployments, dest) + err = deployments.CopyDeploymentFiles(tt.deployType, draftConfig, templateWriter) + if tt.shouldError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + + generatedContent, err := os.ReadFile(tt.tempFileName) + assert.Nil(t, err) + + if _, err := os.Stat(tt.fixturePath); os.IsNotExist(err) { + t.Errorf("Fixture file does not exist at path: %s", tt.fixturePath) + } + + err = fixtures.ValidateContentAgainstFixture(generatedContent, tt.fixturePath) + assert.Nil(t, err) + } + + tt.cleanUp() + }) + } +} + +func TestLoadConfig(t *testing.T) { + fakeFS, err := createMockDeploymentTemplatesFS() + assert.Nil(t, err) + + d, err := createMockDeployments("deployments", fakeFS) + assert.Nil(t, err) + + cases := []loadConfTestCase{ + {"helm", true}, + {"unsupported", false}, + } + + for _, c := range cases { + if c.isNil { + _, err = d.loadConfig(c.deployType) + assert.Nil(t, err) + } else { + _, err = d.loadConfig(c.deployType) + assert.NotNil(t, err) + } + } +} + +func TestPopulateConfigs(t *testing.T) { + fakeFS, err := createMockDeploymentTemplatesFS() + assert.Nil(t, err) + + d, err := createMockDeployments("deployments", fakeFS) + assert.Nil(t, err) + + d.PopulateConfigs() + assert.Equal(t, 3, len(d.configs)) + + d, err = createTestDeploymentEmbed("deployments") + assert.Nil(t, err) + + d.PopulateConfigs() + assert.Equal(t, 3, len(d.configs)) +} + +type loadConfTestCase struct { + deployType string + isNil bool +} + +func createTempDeploymentFile(dirPath, fileName, path string) error { + err := os.MkdirAll(dirPath, 0755) + if err != nil { + return err + } + file, err := os.Create(fileName) + if err != nil { + return err + } + fmt.Printf("file %v\n", file) + defer file.Close() + + var source *os.File + source, err = os.Open(path) + if err != nil { + return err + } + fmt.Printf("source %v\n", source) + defer source.Close() + + _, err = io.Copy(file, source) + if err != nil { + return err + } + return nil +} + +func createMockDeploymentTemplatesFS() (fs.FS, error) { + rootPath := "deplyments/" + embedFiles, err := embedutils.EmbedFStoMapWithFiles(template.Deployments, "deployments") + if err != nil { + return nil, fmt.Errorf("failed to readDir: %w in embeded files", err) + } + + mockFS := fstest.MapFS{} + + for path, file := range embedFiles { + if file.IsDir() { + mockFS[path] = &fstest.MapFile{Mode: fs.ModeDir} + } else { + bytes, err := template.Deployments.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failes to read file: %w", err) + } + mockFS[path] = &fstest.MapFile{Data: bytes} + } + } + + mockFS[rootPath+"emptyDir"] = &fstest.MapFile{Mode: fs.ModeDir} + mockFS[rootPath+"corrupted"] = &fstest.MapFile{Mode: fs.ModeDir} + mockFS[rootPath+"corrupted/draft.yaml"] = &fstest.MapFile{Data: []byte("fake yaml data")} + + return mockFS, nil +} + +func createMockDeployments(dirPath string, mockDeployments fs.FS) (*Deployments, error) { + dest := "." + + deployMap, err := fsToMap(mockDeployments, dirPath) + if err != nil { + return nil, fmt.Errorf("failed fsToMap: %w", err) + } + + d := &Deployments{ + deploys: deployMap, + dest: dest, + configs: make(map[string]*config.DraftConfig), + deploymentTemplates: mockDeployments, + } + + return d, nil +} + +func createTestDeploymentEmbed(dirPath string) (*Deployments, error) { + dest := "." + + deployMap, err := embedutils.EmbedFStoMap(template.Deployments, "deployments") + if err != nil { + return nil, fmt.Errorf("failed to create deployMap: %w", err) + } + + d := &Deployments{ + deploys: deployMap, + dest: dest, + configs: make(map[string]*config.DraftConfig), + deploymentTemplates: template.Deployments, + } + + return d, nil +} + +func fsToMap(fsFS fs.FS, path string) (map[string]fs.DirEntry, error) { + files, err := fs.ReadDir(fsFS, path) + if err != nil { + return nil, fmt.Errorf("failed to ReadDir: %w", err) + } + + mapping := make(map[string]fs.DirEntry) + + for _, f := range files { + if f.IsDir() { + mapping[f.Name()] = f + } + } + + return mapping, nil +} From 04dabdad0bd6ec8fb79aebe0a4a0a4ecfb99c600 Mon Sep 17 00:00:00 2001 From: Brandon Foley Date: Fri, 4 Oct 2024 11:14:50 -0400 Subject: [PATCH 3/6] adding tests/validation --- pkg/config/draftconfig.go | 10 ++ pkg/config/draftconfig_template_test.go | 49 +++++++ pkg/config/draftconfig_test.go | 168 ++++++++++++++++++++++++ 3 files changed, 227 insertions(+) diff --git a/pkg/config/draftconfig.go b/pkg/config/draftconfig.go index 63d13e85..861064e5 100644 --- a/pkg/config/draftconfig.go +++ b/pkg/config/draftconfig.go @@ -133,6 +133,15 @@ func (d *DraftConfig) ApplyDefaultVariablesForVersion(version string) error { return fmt.Errorf("invalid version: %w", err) } + expectedConfigVersionRange, err := semver.ParseRange(d.Versions) + if err != nil { + return fmt.Errorf("invalid config version range: %w", err) + } + + if !expectedConfigVersionRange(v) { + return fmt.Errorf("version %s is outside of config version range %s", version, d.Versions) + } + for _, variable := range d.Variables { if variable.Value == "" { expectedRange, err := semver.ParseRange(variable.Versions) @@ -150,6 +159,7 @@ func (d *DraftConfig) ApplyDefaultVariablesForVersion(version string) error { if err != nil { return fmt.Errorf("apply default variables: %w", err) } + defaultVal, err := d.recurseReferenceVars(referenceVar, referenceVar, true) if err != nil { return fmt.Errorf("apply default variables: %w", err) diff --git a/pkg/config/draftconfig_template_test.go b/pkg/config/draftconfig_template_test.go index 89a2d504..479dcdb6 100644 --- a/pkg/config/draftconfig_template_test.go +++ b/pkg/config/draftconfig_template_test.go @@ -96,6 +96,13 @@ func loadTemplatesWithValidation() error { return fmt.Errorf("template %s has an invalid type: %s", path, currTemplate.Type) } + // version range check once we define versions + // if _, err := semver.ParseRange(currTemplate.Versions); err != nil { + // return fmt.Errorf("template %s has an invalid version range: %s", path, currTemplate.Versions) + // } + + referenceVarMap := map[string]*BuilderVar{} + allVariables := map[string]*BuilderVar{} for _, variable := range currTemplate.Variables { if variable.Name == "" { return fmt.Errorf("template %s has a variable with no name", path) @@ -108,9 +115,51 @@ func loadTemplatesWithValidation() error { if _, ok := validVariableKinds[variable.Kind]; !ok { return fmt.Errorf("template %s has an invalid variable kind: %s", path, variable.Kind) } + + // version range check once we define versions + // if _, err := semver.ParseRange(variable.Versions); err != nil { + // return fmt.Errorf("template %s has an invalid version range: %s", path, variable.Versions) + // } + + allVariables[variable.Name] = variable + if variable.Default.ReferenceVar != "" { + referenceVarMap[variable.Name] = variable + } + } + + for _, currVar := range referenceVarMap { + refVar, ok := allVariables[currVar.Default.ReferenceVar] + if !ok { + return fmt.Errorf("template %s has a variable %s with reference to a non-existent variable: %s", path, currVar.Name, currVar.Default.ReferenceVar) + } + + if currVar.Name == refVar.Name { + return fmt.Errorf("template %s has a variable with cyclical reference to itself: %s", path, currVar.Name) + } + + if isCyclicalVariableReference(currVar, refVar, allVariables, true) { + return fmt.Errorf("template %s has a variable with cyclical reference to itself: %s", path, currVar.Name) + } } allTemplates[strings.ToLower(currTemplate.TemplateName)] = currTemplate return nil }) } + +func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariables map[string]*BuilderVar, isFirst bool) bool { + if !isFirst && initialVar.Name == currRefVar.Name { + return true + } + + if currRefVar.Default.ReferenceVar == "" { + return false + } + + refVar, ok := allVariables[currRefVar.Default.ReferenceVar] + if !ok { + return false + } + + return isCyclicalVariableReference(initialVar, refVar, allVariables, false) +} diff --git a/pkg/config/draftconfig_test.go b/pkg/config/draftconfig_test.go index fddf83c5..22c8e223 100644 --- a/pkg/config/draftconfig_test.go +++ b/pkg/config/draftconfig_test.go @@ -197,3 +197,171 @@ func TestApplyDefaultVariables(t *testing.T) { }) } } + +func TestApplyDefaultVariablesForVersion(t *testing.T) { + tests := []struct { + testName string + version string + draftConfig DraftConfig + customInputs map[string]string + want map[string]string + wantErrMsg string + }{ + { + testName: "excludeOutOfVersionRangeVariables", + version: "0.0.1", + draftConfig: DraftConfig{ + Versions: ">=0.0.1 <=0.0.2", + Variables: []*BuilderVar{ + { + Name: "var1", + Value: "", + Versions: ">=0.0.1", + Default: BuilderVarDefault{ + Value: "default-value-1", + }, + }, + { + Name: "var2", + Value: "custom-value-2", + Versions: ">=0.0.2", + Default: BuilderVarDefault{ + Value: "default-value-2", + }, + }, + }, + }, + want: map[string]string{ + "var1": "default-value-1", + }, + }, + { + testName: "emptyInputVersion", + version: "", + draftConfig: DraftConfig{ + Versions: ">=0.0.1 <=0.0.2", + Variables: []*BuilderVar{ + { + Name: "var1", + Value: "", + Versions: ">=0.0.1", + Default: BuilderVarDefault{ + Value: "default-value-1", + }, + }, + { + Name: "var2", + Value: "", + Versions: ">=0.0.2", + Default: BuilderVarDefault{ + Value: "default-value-2", + }, + }, + }, + }, + wantErrMsg: "invalid version: Version string empty", + }, + { + testName: "inputVersionOutOfRange", + version: "0.0.3", + draftConfig: DraftConfig{ + Versions: ">=0.0.1 <=0.0.2", + Variables: []*BuilderVar{ + { + Name: "var1", + Value: "", + Versions: ">=0.0.1", + Default: BuilderVarDefault{ + Value: "default-value-1", + }, + }, + { + Name: "var2", + Value: "", + Versions: ">=0.0.2", + Default: BuilderVarDefault{ + Value: "default-value-2", + }, + }, + }, + }, + wantErrMsg: "version 0.0.3 is outside of config version range >=0.0.1 <=0.0.2", + }, + { + testName: "overwriteDevfaultValue", + version: "0.0.2", + draftConfig: DraftConfig{ + Versions: ">=0.0.1 <=0.0.2", + Variables: []*BuilderVar{ + { + Name: "var1", + Value: "custom-value-1", + Versions: ">=0.0.1", + Default: BuilderVarDefault{ + Value: "default-value-1", + }, + }, + { + Name: "var2", + Value: "custom-value-2", + Versions: ">=0.0.2", + Default: BuilderVarDefault{ + Value: "default-value-2", + }, + }, + }, + }, + want: map[string]string{ + "var1": "custom-value-1", + "var2": "custom-value-2", + }, + }, + { + testName: "referenceVarOverwrite", + version: "0.0.2", + draftConfig: DraftConfig{ + Versions: ">=0.0.1 <=0.0.2", + Variables: []*BuilderVar{ + { + Name: "var1", + Value: "", + Versions: ">=0.0.1", + Default: BuilderVarDefault{ + ReferenceVar: "var2", + }, + }, + { + Name: "var2", + Value: "custom-value-2", + Versions: ">=0.0.2", + Default: BuilderVarDefault{ + Value: "default-value-2", + }, + }, + }, + }, + want: map[string]string{ + "var1": "custom-value-2", + "var2": "custom-value-2", + }, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + if err := tt.draftConfig.ApplyDefaultVariablesForVersion(tt.version); err != nil && err.Error() != tt.wantErrMsg { + t.Error(err) + } else { + for k, v := range tt.want { + variable, err := tt.draftConfig.GetVariable(k) + if err != nil { + t.Error(err) + } + + if variable.Value != v { + t.Errorf("got: %s, want: %s, for test: %s", variable.Value, v, tt.testName) + } + } + } + }) + } +} From e96bf2553064b37aab906694005d56685d173143 Mon Sep 17 00:00:00 2001 From: Brandon Foley Date: Fri, 4 Oct 2024 15:42:55 -0400 Subject: [PATCH 4/6] address comments --- pkg/config/draftconfig_template_test.go | 11 ++++++++--- pkg/handlers/template.go | 2 +- pkg/handlers/template_utils.go | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/config/draftconfig_template_test.go b/pkg/config/draftconfig_template_test.go index 479dcdb6..56ff024e 100644 --- a/pkg/config/draftconfig_template_test.go +++ b/pkg/config/draftconfig_template_test.go @@ -137,7 +137,7 @@ func loadTemplatesWithValidation() error { return fmt.Errorf("template %s has a variable with cyclical reference to itself: %s", path, currVar.Name) } - if isCyclicalVariableReference(currVar, refVar, allVariables, true) { + if isCyclicalVariableReference(currVar, refVar, allVariables, true, map[string]bool{}) { return fmt.Errorf("template %s has a variable with cyclical reference to itself: %s", path, currVar.Name) } } @@ -147,11 +147,15 @@ func loadTemplatesWithValidation() error { }) } -func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariables map[string]*BuilderVar, isFirst bool) bool { +func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariables map[string]*BuilderVar, isFirst bool, visited map[string]bool) bool { if !isFirst && initialVar.Name == currRefVar.Name { return true } + if _, ok := visited[currRefVar.Name]; ok { + return true + } + if currRefVar.Default.ReferenceVar == "" { return false } @@ -161,5 +165,6 @@ func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariable return false } - return isCyclicalVariableReference(initialVar, refVar, allVariables, false) + visited[currRefVar.Name] = true + return isCyclicalVariableReference(initialVar, refVar, allVariables, false, visited) } diff --git a/pkg/handlers/template.go b/pkg/handlers/template.go index 7f20e1b9..c6f81bf2 100644 --- a/pkg/handlers/template.go +++ b/pkg/handlers/template.go @@ -4,13 +4,13 @@ import ( "bytes" "fmt" "io/fs" - "log" "path/filepath" "strings" tmpl "text/template" "github.com/Azure/draft/pkg/config" "github.com/Azure/draft/pkg/templatewriter" + log "github.com/sirupsen/logrus" ) type Template struct { diff --git a/pkg/handlers/template_utils.go b/pkg/handlers/template_utils.go index 9c073460..22922be5 100644 --- a/pkg/handlers/template_utils.go +++ b/pkg/handlers/template_utils.go @@ -3,13 +3,13 @@ package handlers import ( "fmt" "io/fs" - "log" "path/filepath" "strings" "github.com/Azure/draft/pkg/config" "github.com/Azure/draft/template" "github.com/blang/semver/v4" + log "github.com/sirupsen/logrus" ) var templateConfigs map[string]*Template From 2f66798afd555ce656a4fccf638292cd9426d0a8 Mon Sep 17 00:00:00 2001 From: Brandon Foley Date: Mon, 7 Oct 2024 13:00:20 -0400 Subject: [PATCH 5/6] address comments --- pkg/config/draftconfig_template_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/config/draftconfig_template_test.go b/pkg/config/draftconfig_template_test.go index 56ff024e..c87216ed 100644 --- a/pkg/config/draftconfig_template_test.go +++ b/pkg/config/draftconfig_template_test.go @@ -137,7 +137,7 @@ func loadTemplatesWithValidation() error { return fmt.Errorf("template %s has a variable with cyclical reference to itself: %s", path, currVar.Name) } - if isCyclicalVariableReference(currVar, refVar, allVariables, true, map[string]bool{}) { + if isCyclicalVariableReference(currVar, refVar, allVariables, map[string]bool{}) { return fmt.Errorf("template %s has a variable with cyclical reference to itself: %s", path, currVar.Name) } } @@ -147,8 +147,8 @@ func loadTemplatesWithValidation() error { }) } -func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariables map[string]*BuilderVar, isFirst bool, visited map[string]bool) bool { - if !isFirst && initialVar.Name == currRefVar.Name { +func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariables map[string]*BuilderVar, visited map[string]bool) bool { + if initialVar.Name == currRefVar.Name { return true } From 469573c129f6de2ac96e36fb07bd363e7cb74c03 Mon Sep 17 00:00:00 2001 From: Brandon Foley Date: Mon, 7 Oct 2024 13:07:10 -0400 Subject: [PATCH 6/6] test fix --- pkg/config/draftconfig_template_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/draftconfig_template_test.go b/pkg/config/draftconfig_template_test.go index c87216ed..5a48d036 100644 --- a/pkg/config/draftconfig_template_test.go +++ b/pkg/config/draftconfig_template_test.go @@ -166,5 +166,5 @@ func isCyclicalVariableReference(initialVar, currRefVar *BuilderVar, allVariable } visited[currRefVar.Name] = true - return isCyclicalVariableReference(initialVar, refVar, allVariables, false, visited) + return isCyclicalVariableReference(initialVar, refVar, allVariables, visited) }