Terrafile Community Format Support (#6)

* Replace CircleCI with Buildkite
* go mod tidy && go mod vendor
* Add community Terrafile format support
* Update README
* Fallback to community format only in case of yaml errors
* Remove Buildkite integration as this repo is public

Co-authored-by: Anthony Vylushchak <[email protected]>

.gitignore

@@ -1,3 +1,6 @@
+.idea
+.vscode
+
 # Binaries for programs and plugins
 *.exe
 *.exe~

README.md

@@ -19,41 +19,79 @@ Download your preferred flavor from the [releases](https://github.com/coretech/t
 
 Terrafile expects a file named `Terrafile` which will contain your terraform module dependencies in a yaml like format.
 
-An example Terrafile:
+Terrafile config file in custom directory
 
+```sh
+$ terrafile -f config/Terrafile
 ```
-[email protected]:segmentio/my-modules:
-     - chamber_v2.0.0
-     - constants_v1.0.1
-     - iam_v1.0.0
-     - service_v1.0.4
-     - rds_v0.0.5
-     - worker_v1.1.0
-     - master
+
+Terraform modules exported to custom directory
+
+```sh
+$ terrafile -p /path/to/custom_directory
 ```
 
-Terrafile config file in current directory and modules exported to .terrafile/<user>/<repo>/<ref>
+## Segment Terrafile Format
+
+Segment Terrafile is an internal format that specifies sources with a list of its versions. Each version is vendored under `.terrafile/<user>/<repo>/<ref>`
+
+> :warning: the major downside of using this format is that vendored modules has version in their path. So once you want to upgrade module version, you will need to modify both `Terrafile` and all module usages. Check out [Community Terrafile Format](#community-terrafile-format) which doesn't have such downside
+
+An example Terrafile:
+
+```yaml
+[email protected]:segmentio/terracode-modules:
+  - chamber_v2.0.0
+  - iam_v1.0.0
+  - master
+```
 
 ```sh
 $ terrafile
-[*] Cloning   [email protected]:segmentio/my-modules
+[*] Cloning   [email protected]:segmentio/terracode-modules
 [*] Vendoring ref chamber_v2.0.0
-[*] Vendoring ref constants_v1.0.1
 [*] Vendoring ref iam_v1.0.0
-[*] Vendoring ref service_v1.0.4
-[*] Vendoring ref rds_v0.0.5
-[*] Vendoring ref task-role_v1.0.5
 [*] Vendoring ref master
 ```
 
-Terrafile config file in custom directory
-
 ```sh
-$ terrafile -f config/Terrafile
+$ ls -A1 .terrafile/segmentio/terracode-modules
+chamber_v2.0.0
+iam_v1.0.0
+master
 ```
 
-Terraform modules exported to custom directory
+## Community Terrafile Format
+
+Community Terrafile is a community supported format implemented in various languages (e.g. [github: coretech/terrafile](https://github.com/coretech/terrafile), [npm: terrafile](https://www.npmjs.com/package/terrafile)). Each version is vendored under `.terrafile/<alias>`
+
+In comparison with [Segment Terrafile Format](#segment-terrafile-format), module versions are not included in the vendored path. So once you want to upgrade module version, only `Terrafile` needs to be updated and all module usages will be left untouched.
+
+An example Terrafile:
+
+```yaml
+terracode-modules-chamber:
+  source: "[email protected]:segmentio/terracode-modules"
+  version: "chamber_v2.0.0"
+terracode-modules-iam:
+  source: "[email protected]:segmentio/terracode-modules"
+  version: "iam_v1.0.0"
+terracode-modules:
+  source: "[email protected]:segmentio/terracode-modules"
+  version: "master"
+```
 
 ```sh
-$ terrafile -p /path/to/custom_directory
+$ terrafile
+[*] Cloning   [email protected]:segmentio/terracode-modules
+[*] Vendoring ref chamber_v2.0.0
+[*] Vendoring ref iam_v1.0.0
+[*] Vendoring ref master
+```
+
+```sh
+$ ls -A1 .terrafile
+terracode-modules-chamber
+terracode-modules-iam
+terracode-modules
 ```

main.go

@@ -5,13 +5,11 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
 	"github.com/jessevdk/go-flags"
 	"github.com/nritholtz/stdemuxerhook"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
 )
 
 var opts struct {
@@ -84,7 +82,7 @@ func gitCheckoutRef(repositoryPath string, ref string, destinationDir string) {
 }
 
 func main() {
-	//fmt.Printf("Terrafile: version %v, commit %v, built at %v \n", version, commit, date)
+	// fmt.Printf("Terrafile: version %v, commit %v, built at %v \n", version, commit, date)
 	_, err := flags.Parse(&opts)
 
 	// Invalid choice
@@ -98,27 +96,32 @@ func main() {
 		panic(err)
 	}
 
-	// Parse File
-	var config map[string][]string
-	if err := yaml.Unmarshal(yamlFile, &config); err != nil {
+	// Parse Terrafile
+	sourceDependenciesMap, err := parseTerrafile(yamlFile)
+	if err != nil {
+		panic(err)
+	}
+
+	// Cleanup module path
+	if err := os.RemoveAll(opts.ModulePath); err != nil {
 		panic(err)
 	}
 
-	// Clone modules
-	os.RemoveAll(opts.ModulePath)
-	os.MkdirAll(opts.ModulePath, os.ModePerm)
-	for source, refs := range config {
+	if err := os.MkdirAll(opts.ModulePath, os.ModePerm); err != nil {
+		panic(err)
+	}
+
+	for source, dependencies := range sourceDependenciesMap {
 		fmt.Printf("[*] Cloning   %s\n", source)
 		repo := gitClone(source)
-		for _, ref := range refs {
-			pathParts := strings.Split(source, ":")
-			repositoryName := pathParts[1]
-			fmt.Printf("[*] Vendoring ref %s\n", ref)
-			targetPath, err := filepath.Abs(fmt.Sprintf("%s/%s/%s", opts.ModulePath, repositoryName, ref))
+		for _, dependency := range dependencies {
+			fmt.Printf("[*] Vendoring ref %s\n", dependency.Version)
+			targetPath, err := dependency.GetTargetPath(opts.ModulePath)
 			if err != nil {
 				panic(err)
 			}
-			gitCheckoutRef(repo, ref, targetPath)
+
+			gitCheckoutRef(repo, dependency.Version, targetPath)
 		}
 	}
 }

main_test.go

@@ -22,33 +22,61 @@ func init() {
 	}
 	terrafileBinaryPath = workingDirectory + "/terrafile"
 }
-func TestTerraformWithTerrafilePath(t *testing.T) {
-	folder, back := setup(t)
-	defer back()
 
-	testcli.Run(terrafileBinaryPath, "-f", fmt.Sprint(folder, "/Terrafile"))
-
-	if !testcli.Success() {
-		t.Fatalf("Expected to succeed, but failed: %q with message: %q", testcli.Error(), testcli.Stderr())
-	}
-	// Assert output
-	for _, output := range []string{
-		"Cloning   [email protected]:terraform-aws-modules/terraform-aws-vpc",
-		"Vendoring ref master",
-		"Vendoring ref v1.46.0",
-	} {
-		assert.Contains(t, testcli.Stdout(), output)
+func TestTerrafile(t *testing.T) {
+	tests := []*struct {
+		Description      string
+		TerrafileCreator terrafileCreator
+		ExpectedModules  []string
+	}{
+		{
+			Description:      "Segment Terrafile Format",
+			TerrafileCreator: createSegmentTerrafile,
+			ExpectedModules: []string{
+				"terraform-aws-modules/terraform-aws-vpc/master",
+				"terraform-aws-modules/terraform-aws-vpc/v1.46.0",
+			},
+		},
+		{
+			Description:      "Community Terrafile Format",
+			TerrafileCreator: createCommunityTerrafile,
+			ExpectedModules: []string{
+				"terraform-aws-vpc",
+				"terraform-aws-vpc-1.46.0",
+			},
+		},
 	}
-	// Assert files exist
-	for _, moduleName := range []string{
-		"terraform-aws-modules/terraform-aws-vpc/master",
-		"terraform-aws-modules/terraform-aws-vpc/v1.46.0",
-	} {
-		assert.DirExists(t, path.Join(workingDirectory, "./.terrafile", moduleName))
+
+	for _, test := range tests {
+		t.Run(test.Description, func(t *testing.T) {
+			folder, back := setup(t, test.TerrafileCreator)
+			defer back()
+			defer func() {
+				assert.NoError(t, os.RemoveAll(path.Join(workingDirectory, "./.terrafile")))
+			}()
+
+			testcli.Run(terrafileBinaryPath, "-d", "-f", fmt.Sprint(folder, "/Terrafile"))
+
+			if !testcli.Success() {
+				t.Fatalf("Expected to succeed, but failed: %q \nStdout: %q \nStderr: %q", testcli.Error(), testcli.Stdout(), testcli.Stderr())
+			}
+			// Assert output
+			for _, output := range []string{
+				"Cloning   [email protected]:terraform-aws-modules/terraform-aws-vpc",
+				"Vendoring ref master",
+				"Vendoring ref v1.46.0",
+			} {
+				assert.Contains(t, testcli.Stdout(), output)
+			}
+			// Assert files exist
+			for _, moduleName := range test.ExpectedModules {
+				assert.DirExists(t, path.Join(workingDirectory, "./.terrafile", moduleName))
+			}
+		})
 	}
 }
 
-func setup(t *testing.T) (current string, back func()) {
+func setup(t *testing.T, createTerrafile terrafileCreator) (current string, back func()) {
 	folder, err := ioutil.TempDir("", "")
 	assert.NoError(t, err)
 	createTerrafile(t, folder)
@@ -61,10 +89,23 @@ func createFile(t *testing.T, filename string, contents string) {
 	assert.NoError(t, ioutil.WriteFile(filename, []byte(contents), 0644))
 }
 
-func createTerrafile(t *testing.T, folder string) {
+type terrafileCreator func(t *testing.T, folder string)
+
+func createSegmentTerrafile(t *testing.T, folder string) {
 	var yaml = `[email protected]:terraform-aws-modules/terraform-aws-vpc:
   - v1.46.0
   - master
 `
 	createFile(t, path.Join(folder, "Terrafile"), yaml)
 }
+
+func createCommunityTerrafile(t *testing.T, folder string) {
+	var yaml = `terraform-aws-vpc:
+  source: "[email protected]:terraform-aws-modules/terraform-aws-vpc"
+  version: master
+terraform-aws-vpc-1.46.0:
+  source: "[email protected]:terraform-aws-modules/terraform-aws-vpc"
+  version: v1.46.0
+`
+	createFile(t, path.Join(folder, "Terrafile"), yaml)
+}