Autogenerated documentation for bundle config (#2033)

## Changes

Documentation autogeneration tool. This tool uses same annotations_*.yml
files as in json-schema

Result will go
[there](https://docs.databricks.com/en/dev-tools/bundles/reference.html)
and
[there](https://docs.databricks.com/en/dev-tools/bundles/resources.html#cluster)

## Tests
Manually

Makefile

@@ -48,6 +48,9 @@ vendor:
 schema:
 	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
+docs:
+	go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen
+
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
 
 integration:
@@ -56,4 +59,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs

bundle/docsgen/README.md

@@ -0,0 +1,79 @@
+## docs-autogen
+
+1. Install [Golang](https://go.dev/doc/install)
+2. Run `make vendor docs` from the repo
+3. See generated documents in `./bundle/docsgen/output` directory
+4. To change descriptions update content in `./bundle/internal/schema/annotations.yml` or `./bundle/internal/schema/annotations_openapi_overrides.yml` and re-run `make docs`
+
+For simpler usage run it together with copy command to move resulting files to local `docs` repo. Note that it will overwrite any local changes in affected files. Example:
+
+```
+make docs && cp bundle/docgen/output/*.md ../docs/source/dev-tools/bundles
+```
+
+To change intro sections for files update them in `templates/` directory
+
+### Annotation file structure
+
+```yaml
+"<root-type-name>":
+  "<property-name>":
+    description: Description of the property, only plain text is supported
+    markdown_description: Description with markdown support, if defined it will override the value in docs and in JSON-schema
+    markdown_examples: Custom block for any example, in free form, Markdown is supported
+    title: JSON-schema title, not used in docs
+    default: Default value of the property, not used in docs
+    enum: Possible values of enum-type, not used in docs
+```
+
+Descriptions with `PLACEHOLDER` value are not displayed in docs and JSON-schema
+
+All relative links like `[_](/dev-tools/bundles/settings.md#cluster_id)` are kept as is in docs but converted to absolute links in JSON schema
+
+To change description for type itself (not its fields) use `"_"`:
+
+```yaml
+github.com/databricks/cli/bundle/config/resources.Cluster:
+  "_":
+    "markdown_description": |-
+      The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
+```
+
+### Example annotation
+
+```yaml
+github.com/databricks/cli/bundle/config.Bundle:
+  "cluster_id":
+    "description": |-
+      The ID of a cluster to use to run the bundle.
+    "markdown_description": |-
+      The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
+  "compute_id":
+    "description": |-
+      PLACEHOLDER
+  "databricks_cli_version":
+    "description": |-
+      The Databricks CLI version to use for the bundle.
+    "markdown_description": |-
+      The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version).
+  "deployment":
+    "description": |-
+      The definition of the bundle deployment
+    "markdown_description": |-
+      The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md).
+  "git":
+    "description": |-
+      The Git version control details that are associated with your bundle.
+    "markdown_description": |-
+      The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git).
+  "name":
+    "description": |-
+      The name of the bundle.
+  "uuid":
+    "description": |-
+      PLACEHOLDER
+```
+
+### TODO
+
+Add file watcher to track changes in the annotation files and re-run `make docs` script automtically

bundle/docsgen/main.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/internal/annotation"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+const (
+	rootFileName      = "reference.md"
+	resourcesFileName = "resources.md"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println("Usage: go run main.go <annotation-file> <output-file>")
+		os.Exit(1)
+	}
+
+	annotationDir := os.Args[1]
+	docsDir := os.Args[2]
+	outputDir := path.Join(docsDir, "output")
+	templatesDir := path.Join(docsDir, "templates")
+
+	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(outputDir, 0o755); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	rootHeader, err := os.ReadFile(path.Join(templatesDir, rootFileName))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = generateDocs(
+		[]string{path.Join(annotationDir, "annotations.yml")},
+		path.Join(outputDir, rootFileName),
+		reflect.TypeOf(config.Root{}),
+		string(rootHeader),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	resourcesHeader, err := os.ReadFile(path.Join(templatesDir, resourcesFileName))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = generateDocs(
+		[]string{path.Join(annotationDir, "annotations_openapi.yml"), path.Join(annotationDir, "annotations_openapi_overrides.yml"), path.Join(annotationDir, "annotations.yml")},
+		path.Join(outputDir, resourcesFileName),
+		reflect.TypeOf(config.Resources{}),
+		string(resourcesHeader),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func generateDocs(inputPaths []string, outputPath string, rootType reflect.Type, header string) error {
+	annotations, err := annotation.LoadAndMerge(inputPaths)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// schemas is used to resolve references to schemas
+	schemas := map[string]*jsonschema.Schema{}
+	// ownFields is used to track fields that are defined in the annotation file and should be included in the docs page
+	ownFields := map[string]bool{}
+
+	s, err := jsonschema.FromType(rootType, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
+		func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+			_, isOwnField := annotations[jsonschema.TypePath(typ)]
+			if isOwnField {
+				ownFields[jsonschema.TypePath(typ)] = true
+			}
+
+			refPath := getPath(typ)
+			shouldHandle := strings.HasPrefix(refPath, "github.com")
+			if !shouldHandle {
+				schemas[jsonschema.TypePath(typ)] = &s
+				return s
+			}
+
+			a := annotations[refPath]
+			if a == nil {
+				a = map[string]annotation.Descriptor{}
+			}
+
+			rootTypeAnnotation, ok := a["_"]
+			if ok {
+				assignAnnotation(&s, rootTypeAnnotation)
+			}
+
+			for k, v := range s.Properties {
+				assignAnnotation(v, a[k])
+			}
+
+			schemas[jsonschema.TypePath(typ)] = &s
+			return s
+		},
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	nodes := buildNodes(s, schemas, ownFields)
+	err = buildMarkdown(nodes, outputPath, header)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func getPath(typ reflect.Type) string {
+	return typ.PkgPath() + "." + typ.Name()
+}
+
+func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
+	if a.Description != "" && a.Description != annotation.Placeholder {
+		s.Description = a.Description
+	}
+	if a.MarkdownDescription != "" {
+		s.MarkdownDescription = a.MarkdownDescription
+	}
+	if a.MarkdownExamples != "" {
+		s.Examples = []any{a.MarkdownExamples}
+	}
+}

bundle/docsgen/markdown.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+)
+
+func buildMarkdown(nodes []rootNode, outputFile, header string) error {
+	m := newMardownRenderer()
+	m = m.PlainText(header)
+	for _, node := range nodes {
+		m = m.LF()
+		if node.TopLevel {
+			m = m.H2(node.Title)
+		} else {
+			m = m.H3(node.Title)
+		}
+		m = m.LF()
+
+		if node.Type != "" {
+			m = m.PlainText(fmt.Sprintf("**`Type: %s`**", node.Type))
+			m = m.LF()
+		}
+		m = m.PlainText(node.Description)
+		m = m.LF()
+
+		if len(node.ObjectKeyAttributes) > 0 {
+			n := pickLastWord(node.Title)
+			n = removePluralForm(n)
+			m = m.CodeBlocks("yaml", fmt.Sprintf("%ss:\n  <%s-name>:\n    <%s-field-name>: <%s-field-value>", n, n, n, n))
+			m = m.LF()
+			m = buildAttributeTable(m, node.ObjectKeyAttributes)
+		} else if len(node.ArrayItemAttributes) > 0 {
+			m = m.LF()
+			m = buildAttributeTable(m, node.ArrayItemAttributes)
+		} else if len(node.Attributes) > 0 {
+			m = m.LF()
+			m = buildAttributeTable(m, node.Attributes)
+		}
+
+		if node.Example != "" {
+			m = m.LF()
+			m = m.PlainText("**Example**")
+			m = m.LF()
+			m = m.PlainText(node.Example)
+		}
+	}
+
+	f, err := os.Create(outputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = f.WriteString(m.String())
+	if err != nil {
+		log.Fatal(err)
+	}
+	return f.Close()
+}
+
+func pickLastWord(s string) string {
+	words := strings.Split(s, ".")
+	return words[len(words)-1]
+}
+
+// Build a custom table which we use in Databricks website
+func buildAttributeTable(m *markdownRenderer, attributes []attributeNode) *markdownRenderer {
+	m = m.LF()
+	m = m.PlainText(".. list-table::")
+	m = m.PlainText("   :header-rows: 1")
+	m = m.LF()
+
+	m = m.PlainText("   * - Key")
+	m = m.PlainText("     - Type")
+	m = m.PlainText("     - Description")
+	m = m.LF()
+
+	for _, a := range attributes {
+		m = m.PlainText("   * - " + fmt.Sprintf("`%s`", a.Title))
+		m = m.PlainText("     - " + a.Type)
+		m = m.PlainText("     - " + formatDescription(a))
+		m = m.LF()
+	}
+	return m
+}
+
+func formatDescription(a attributeNode) string {
+	s := strings.ReplaceAll(a.Description, "\n", " ")
+	if a.Link != "" {
+		if strings.HasSuffix(s, ".") {
+			s += " "
+		} else if s != "" {
+			s += ". "
+		}
+		s += fmt.Sprintf("See [_](#%s).", a.Link)
+	}
+	return s
+}