[WIP] feat: Config reference documentation autogeneration

Makefile

@@ -33,10 +33,13 @@ snapshot:
 
 vendor:
 	go mod vendor
-  
+
 schema:
 	go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
+docs:
+	go run ./bundle/internal/docs ./bundle/internal/schema ./bundle/internal/docs
+
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
 
 integration:
@@ -45,4 +48,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short docs

bundle/internal/annotation/main.go

@@ -0,0 +1,56 @@
+package annotation
+
+import (
+	"bytes"
+	"os"
+
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/merge"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
+)
+
+type Descriptor struct {
+	Description         string `json:"description,omitempty"`
+	MarkdownDescription string `json:"markdown_description,omitempty"`
+	Title               string `json:"title,omitempty"`
+	Default             any    `json:"default,omitempty"`
+	Enum                []any  `json:"enum,omitempty"`
+	MarkdownExamples    string `json:"markdown_examples,omitempty"`
+}
+
+/**
+ * Parsed file with annotations, expected format:
+ * github.com/databricks/cli/bundle/config.Bundle:
+ *  	cluster_id:
+ *      description: "Description"
+ */
+type File map[string]map[string]Descriptor
+
+func LoadAndMerge(sources []string) (File, error) {
+	prev := dyn.NilValue
+	for _, path := range sources {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
+		if err != nil {
+			return nil, err
+		}
+		prev, err = merge.Merge(prev, generated)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var data File
+
+	err := convert.ToTyped(&data, prev)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+const Placeholder = "PLACEHOLDER"

bundle/internal/docs/.gitignore

@@ -0,0 +1 @@
+output/**/*

bundle/internal/docs/README.md

@@ -0,0 +1,71 @@
+## docs-autogen
+
+1. Install [Golang](https://go.dev/doc/install)
+2. Run `go mod download` from the repo root
+3. Run `make docs` from the repo
+4. See generated documents in `./bundle/internal/docs/output` directory
+5. To change descriptions update content in `./bundle/internal/schema/annotations.yml` or `./bundle/internal/schema/annotations_openapi_overrides.yml` and re-run `make docs`
+
+For simpler usage run it together with copy command to move resulting files to local `docs` repo. Note that it will overwrite any local changes in affected files. Example:
+
+```
+make docs && cp bundle/internal/docs/output/*.md ../docs/source/dev-tools/bundles
+```
+
+To change file names or file headers update them in `main.go` file in this directory
+
+### Annotation file structure
+
+```yaml
+"<root-type-name>":
+  "<property-name>":
+    description: Description of the property, only plain text is supported
+    markdown_description: Description with markdown support, if defined it will override the value in docs and in JSON-schema
+    markdown_examples: Custom block for any example, in free form, Markdown is supported
+    title: JSON-schema title, not used in docs
+    default: Default value of the property, not used in docs
+    enum: Possible values of enum-type, not used in docs
+```
+
+Descriptions with `PLACEHOLDER` value are not displayed in docs and JSON-schema
+
+All relative links like `[_](/dev-tools/bundles/settings.md#cluster_id)` are kept as is in docs but converted to absolute links in JSON schema
+
+### Example annotation
+
+```yaml
+github.com/databricks/cli/bundle/config.Bundle:
+  "cluster_id":
+    "description": |-
+      The ID of a cluster to use to run the bundle.
+    "markdown_description": |-
+      The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
+  "compute_id":
+    "description": |-
+      PLACEHOLDER
+  "databricks_cli_version":
+    "description": |-
+      The Databricks CLI version to use for the bundle.
+    "markdown_description": |-
+      The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version).
+  "deployment":
+    "description": |-
+      The definition of the bundle deployment
+    "markdown_description": |-
+      The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md).
+  "git":
+    "description": |-
+      The Git version control details that are associated with your bundle.
+    "markdown_description": |-
+      The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git).
+  "name":
+    "description": |-
+      The name of the bundle.
+  "uuid":
+    "description": |-
+      PLACEHOLDER
+```
+
+### TODO
+
+Add file watcher to track changes in the annotation files and re-run `make docs` script automtically

bundle/internal/docs/main.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/internal/annotation"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println("Usage: go run main.go <annotation-file> <output-file>")
+		os.Exit(1)
+	}
+
+	annotationDir := os.Args[1]
+	docsDir := os.Args[2]
+	outputDir := path.Join(docsDir, "output")
+
+	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(outputDir, 0o755); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	err := generateDocs(
+		[]string{path.Join(annotationDir, "annotations.yml")},
+		path.Join(outputDir, rootFileName),
+		reflect.TypeOf(config.Root{}),
+		rootHeader,
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = generateDocs(
+		[]string{path.Join(annotationDir, "annotations_openapi.yml"), path.Join(annotationDir, "annotations_openapi_overrides.yml"), path.Join(annotationDir, "annotations.yml")},
+		path.Join(outputDir, resourcesFileName),
+		reflect.TypeOf(config.Resources{}),
+		resourcesHeader,
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func generateDocs(inputPaths []string, outputPath string, rootType reflect.Type, header string) error {
+	annotations, err := annotation.LoadAndMerge(inputPaths)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	schemas := map[string]jsonschema.Schema{}
+	customFields := map[string]bool{}
+
+	s, err := jsonschema.FromType(rootType, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
+		func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+			_, isCustomField := annotations[jsonschema.TypePath(typ)]
+			if isCustomField {
+				customFields[jsonschema.TypePath(typ)] = true
+			}
+
+			refPath := getPath(typ)
+			shouldHandle := strings.HasPrefix(refPath, "github.com")
+			if !shouldHandle {
+				schemas[jsonschema.TypePath(typ)] = s
+				return s
+			}
+
+			a := annotations[refPath]
+			if a == nil {
+				a = map[string]annotation.Descriptor{}
+			}
+
+			rootTypeAnnotation, ok := a["_"]
+			if ok {
+				assignAnnotation(&s, rootTypeAnnotation)
+			}
+
+			for k, v := range s.Properties {
+				assignAnnotation(v, a[k])
+			}
+
+			schemas[jsonschema.TypePath(typ)] = s
+			return s
+		},
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	nodes := getNodes(s, schemas, customFields)
+	err = buildMarkdown(nodes, outputPath, header)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func getPath(typ reflect.Type) string {
+	return typ.PkgPath() + "." + typ.Name()
+}
+
+func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
+	if a.Description != "" && a.Description != annotation.Placeholder {
+		s.Description = a.Description
+	}
+	if a.MarkdownDescription != "" {
+		s.MarkdownDescription = a.MarkdownDescription
+	}
+	if a.MarkdownExamples != "" {
+		s.Examples = []any{a.MarkdownExamples}
+	}
+}