diff --git a/Makefile b/Makefile index 2c84d88ba7..20b868a611 100644 --- a/Makefile +++ b/Makefile @@ -33,10 +33,13 @@ snapshot: vendor: go mod vendor - + schema: go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json +docs: + go run ./bundle/internal/docs ./bundle/internal/schema ./bundle/internal/docs + INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h integration: @@ -45,4 +48,4 @@ integration: integration-short: $(INTEGRATION) -short -.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short +.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short docs diff --git a/bundle/internal/annotation/main.go b/bundle/internal/annotation/main.go new file mode 100644 index 0000000000..0053e16c95 --- /dev/null +++ b/bundle/internal/annotation/main.go @@ -0,0 +1,56 @@ +package annotation + +import ( + "bytes" + "os" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/dyn/yamlloader" +) + +type Descriptor struct { + Description string `json:"description,omitempty"` + MarkdownDescription string `json:"markdown_description,omitempty"` + Title string `json:"title,omitempty"` + Default any `json:"default,omitempty"` + Enum []any `json:"enum,omitempty"` + MarkdownExamples string `json:"markdown_examples,omitempty"` +} + +/** + * Parsed file with annotations, expected format: + * github.com/databricks/cli/bundle/config.Bundle: + * cluster_id: + * description: "Description" + */ +type File map[string]map[string]Descriptor + +func LoadAndMerge(sources []string) (File, error) { + prev := dyn.NilValue + for _, path := range sources { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b)) + if err != nil { + return nil, err + } + prev, err = merge.Merge(prev, generated) + if err != nil { + return nil, err + } + } + + var data File + + err := convert.ToTyped(&data, prev) + if err != nil { + return nil, err + } + return data, nil +} + +const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/docs/.gitignore b/bundle/internal/docs/.gitignore new file mode 100644 index 0000000000..294bc037fc --- /dev/null +++ b/bundle/internal/docs/.gitignore @@ -0,0 +1 @@ +output/**/* diff --git a/bundle/internal/docs/README.md b/bundle/internal/docs/README.md new file mode 100644 index 0000000000..0dd5727f43 --- /dev/null +++ b/bundle/internal/docs/README.md @@ -0,0 +1,71 @@ +## docs-autogen + +1. Install [Golang](https://go.dev/doc/install) +2. Run `go mod download` from the repo root +3. Run `make docs` from the repo +4. See generated documents in `./bundle/internal/docs/output` directory +5. To change descriptions update content in `./bundle/internal/schema/annotations.yml` or `./bundle/internal/schema/annotations_openapi_overrides.yml` and re-run `make docs` + +For simpler usage run it together with copy command to move resulting files to local `docs` repo. Note that it will overwrite any local changes in affected files. Example: + +``` +make docs && cp bundle/internal/docs/output/*.md ../docs/source/dev-tools/bundles +``` + +To change file names or file headers update them in `main.go` file in this directory + +### Annotation file structure + +```yaml +"": + "": + description: Description of the property, only plain text is supported + markdown_description: Description with markdown support, if defined it will override the value in docs and in JSON-schema + markdown_examples: Custom block for any example, in free form, Markdown is supported + title: JSON-schema title, not used in docs + default: Default value of the property, not used in docs + enum: Possible values of enum-type, not used in docs +``` + +Descriptions with `PLACEHOLDER` value are not displayed in docs and JSON-schema + +All relative links like `[_](/dev-tools/bundles/settings.md#cluster_id)` are kept as is in docs but converted to absolute links in JSON schema + +### Example annotation + +```yaml +github.com/databricks/cli/bundle/config.Bundle: + "cluster_id": + "description": |- + The ID of a cluster to use to run the bundle. + "markdown_description": |- + The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id). + "compute_id": + "description": |- + PLACEHOLDER + "databricks_cli_version": + "description": |- + The Databricks CLI version to use for the bundle. + "markdown_description": |- + The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version). + "deployment": + "description": |- + The definition of the bundle deployment + "markdown_description": |- + The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md). + "git": + "description": |- + The Git version control details that are associated with your bundle. + "markdown_description": |- + The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git). + "name": + "description": |- + The name of the bundle. + "uuid": + "description": |- + PLACEHOLDER +``` + +### TODO + +Add file watcher to track changes in the annotation files and re-run `make docs` script automtically diff --git a/bundle/internal/docs/main.go b/bundle/internal/docs/main.go new file mode 100644 index 0000000000..e4e603b1d2 --- /dev/null +++ b/bundle/internal/docs/main.go @@ -0,0 +1,119 @@ +package main + +import ( + "fmt" + "log" + "os" + "path" + "reflect" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/annotation" + "github.com/databricks/cli/libs/jsonschema" +) + +func main() { + if len(os.Args) != 3 { + fmt.Println("Usage: go run main.go ") + os.Exit(1) + } + + annotationDir := os.Args[1] + docsDir := os.Args[2] + outputDir := path.Join(docsDir, "output") + + if _, err := os.Stat(outputDir); os.IsNotExist(err) { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + log.Fatal(err) + } + } + + err := generateDocs( + []string{path.Join(annotationDir, "annotations.yml")}, + path.Join(outputDir, rootFileName), + reflect.TypeOf(config.Root{}), + rootHeader, + ) + if err != nil { + log.Fatal(err) + } + err = generateDocs( + []string{path.Join(annotationDir, "annotations_openapi.yml"), path.Join(annotationDir, "annotations_openapi_overrides.yml"), path.Join(annotationDir, "annotations.yml")}, + path.Join(outputDir, resourcesFileName), + reflect.TypeOf(config.Resources{}), + resourcesHeader, + ) + if err != nil { + log.Fatal(err) + } +} + +func generateDocs(inputPaths []string, outputPath string, rootType reflect.Type, header string) error { + annotations, err := annotation.LoadAndMerge(inputPaths) + if err != nil { + log.Fatal(err) + } + + schemas := map[string]jsonschema.Schema{} + customFields := map[string]bool{} + + s, err := jsonschema.FromType(rootType, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ + func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + _, isCustomField := annotations[jsonschema.TypePath(typ)] + if isCustomField { + customFields[jsonschema.TypePath(typ)] = true + } + + refPath := getPath(typ) + shouldHandle := strings.HasPrefix(refPath, "github.com") + if !shouldHandle { + schemas[jsonschema.TypePath(typ)] = s + return s + } + + a := annotations[refPath] + if a == nil { + a = map[string]annotation.Descriptor{} + } + + rootTypeAnnotation, ok := a["_"] + if ok { + assignAnnotation(&s, rootTypeAnnotation) + } + + for k, v := range s.Properties { + assignAnnotation(v, a[k]) + } + + schemas[jsonschema.TypePath(typ)] = s + return s + }, + }) + if err != nil { + log.Fatal(err) + } + + nodes := getNodes(s, schemas, customFields) + err = buildMarkdown(nodes, outputPath, header) + if err != nil { + log.Fatal(err) + } + return nil +} + +func getPath(typ reflect.Type) string { + return typ.PkgPath() + "." + typ.Name() +} + +func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { + if a.Description != "" && a.Description != annotation.Placeholder { + s.Description = a.Description + } + if a.MarkdownDescription != "" { + s.MarkdownDescription = a.MarkdownDescription + } + if a.MarkdownExamples != "" { + s.Examples = []any{a.MarkdownExamples} + } +} diff --git a/bundle/internal/docs/markdown.go b/bundle/internal/docs/markdown.go new file mode 100644 index 0000000000..508f63578b --- /dev/null +++ b/bundle/internal/docs/markdown.go @@ -0,0 +1,203 @@ +package main + +import ( + "fmt" + "log" + "os" + "strings" + + md "github.com/nao1215/markdown" +) + +const ( + rootFileName = "reference.md" + rootHeader = `--- +description: Configuration reference for databricks.yml +--- + +# Configuration reference + +This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). + +For complete bundle examples, see [_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples). +` +) + +const ( + resourcesFileName = "resources.md" + resourcesHeader = `--- +description: Learn about resources supported by Databricks Asset Bundles and how to configure them. +--- + +# resources + + allows you to specify information about the resources used by the bundle in the ` + "`" + `resources` + "`" + ` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). + +This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [_](/dev-tools/bundles/resource-examples.md). + +## Supported resources + +The following table lists supported resource types for bundles. Some resources can be created by defining them in a bundle and deploying the bundle, and some resources only support referencing an existing resource to include in the bundle. + +Resources are defined using the corresponding [Databricks REST API](/api/workspace/introduction) object's create operation request payload, where the object's supported fields, expressed as YAML, are the resource's supported properties. Links to documentation for each resource's corresponding payloads are listed in the table. + +.. tip:: The ` + "`" + `databricks bundle validate` + "`" + ` command returns warnings if unknown resource properties are found in bundle configuration files. + + +.. list-table:: + :header-rows: 1 + + * - Resource + - Create support + - Corresponding REST API object + + * - [cluster](#cluster) + - ✓ + - [Cluster object](/api/workspace/clusters/create) + + * - [dashboard](#dashboard) + - + - [Dashboard object](/api/workspace/lakeview/create) + + * - [experiment](#experiment) + - ✓ + - [Experiment object](/api/workspace/experiments/createexperiment) + + * - [job](#job) + - ✓ + - [Job object](/api/workspace/jobs/create) + + * - [model (legacy)](#model-legacy) + - ✓ + - [Model (legacy) object](/api/workspace/modelregistry/createmodel) + + * - [model_serving_endpoint](#model-serving-endpoint) + - ✓ + - [Model serving endpoint object](/api/workspace/servingendpoints/create) + + * - [pipeline](#pipeline) + - ✓ + - [Pipeline object]](/api/workspace/pipelines/create) + + * - [quality_monitor](#quality-monitor) + - ✓ + - [Quality monitor object](/api/workspace/qualitymonitors/create) + + * - [registered_model](#registered-model) () + - ✓ + - [Registered model object](/api/workspace/registeredmodels/create) + + * - [schema](#schema) () + - ✓ + - [Schema object](/api/workspace/schemas/create) + + * - [volume](#volume) () + - ✓ + - [Volume object](/api/workspace/volumes/create) +` +) + +func buildMarkdown(nodes []rootNode, outputFile, header string) error { + f, err := os.Create(outputFile) + if err != nil { + log.Fatal(err) + } + defer f.Close() + + m := md.NewMarkdown(f) + m = m.PlainText(header) + for _, node := range nodes { + m = m.LF() + if node.TopLevel { + m = m.H2(node.Title) + } else { + m = m.H3(node.Title) + } + m = m.LF() + + if node.Type != "" { + m = m.PlainText(fmt.Sprintf("**`Type: %s`**", node.Type)) + m = m.LF() + } + m = m.PlainText(node.Description) + m = m.LF() + + if len(node.ObjectKeyAttributes) > 0 { + n := removePluralForm(node.Title) + m = m.CodeBlocks("yaml", fmt.Sprintf("%ss:\n <%s-name>:\n <%s-field-name>: <%s-field-value>", n, n, n, n)) + m = m.LF() + m = buildAttributeTable(m, node.ObjectKeyAttributes) + } else if len(node.ArrayItemAttributes) > 0 { + m = m.LF() + m = buildAttributeTable(m, node.ArrayItemAttributes) + } else if len(node.Attributes) > 0 { + m = m.LF() + m = buildAttributeTable(m, node.Attributes) + } + + if node.Example != "" { + m = m.LF() + m = m.PlainText("**Example**") + m = m.LF() + m = m.PlainText(node.Example) + } + } + + err = m.Build() + if err != nil { + log.Fatal(err) + } + + return nil +} + +// Build a custom table which we use in Databricks website +func buildCustomAttributeTable(m *md.Markdown, attributes []attributeNode) *md.Markdown { + m = m.LF() + m = m.PlainText(".. list-table::") + m = m.PlainText(" :header-rows: 1") + m = m.LF() + + m = m.PlainText(" * - Key") + m = m.PlainText(" - Type") + m = m.PlainText(" - Description") + m = m.LF() + + for _, a := range attributes { + m = m.PlainText(" * - " + fmt.Sprintf("`%s`", a.Title)) + m = m.PlainText(" - " + a.Type) + m = m.PlainText(" - " + formatDescription(a)) + m = m.LF() + } + return m +} + +func buildAttributeTable(m *md.Markdown, attributes []attributeNode) *md.Markdown { + return buildCustomAttributeTable(m, attributes) + + // Rows below are useful for debugging since it renders the table in a regular markdown format + + // rows := [][]string{} + // for _, n := range attributes { + // rows = append(rows, []string{fmt.Sprintf("`%s`", n.Title), n.Type, formatDescription(n.Description)}) + // } + // m = m.CustomTable(md.TableSet{ + // Header: []string{"Key", "Type", "Description"}, + // Rows: rows, + // }, md.TableOptions{AutoWrapText: false, AutoFormatHeaders: false}) + + // return m +} + +func formatDescription(a attributeNode) string { + s := strings.ReplaceAll(a.Description, "\n", " ") + if a.Reference != "" { + if strings.HasSuffix(s, ".") { + s += " " + } else if s != "" { + s += ". " + } + s += fmt.Sprintf("See %s.", md.Link("_", "#"+a.Reference)) + } + return s +} diff --git a/bundle/internal/docs/nodes.go b/bundle/internal/docs/nodes.go new file mode 100644 index 0000000000..4507de08a3 --- /dev/null +++ b/bundle/internal/docs/nodes.go @@ -0,0 +1,175 @@ +package main + +import ( + "sort" + "strings" + + "github.com/databricks/cli/libs/jsonschema" +) + +type rootNode struct { + Title string + Description string + Attributes []attributeNode + Example string + ObjectKeyAttributes []attributeNode + ArrayItemAttributes []attributeNode + TopLevel bool + Type string +} + +type attributeNode struct { + Title string + Type string + Description string + Reference string +} + +type rootProp struct { + k string + v *jsonschema.Schema + topLevel bool +} + +const MapType = "Map" + +func getNodes(s jsonschema.Schema, refs map[string]jsonschema.Schema, customFields map[string]bool) []rootNode { + rootProps := []rootProp{} + for k, v := range s.Properties { + rootProps = append(rootProps, rootProp{k, v, true}) + } + nodes := make([]rootNode, 0, len(rootProps)) + visited := make(map[string]bool) + + for i := 0; i < len(rootProps); i++ { + item := rootProps[i] + k := item.k + v := item.v + + if visited[k] { + continue + } + visited[k] = true + v = resolveRefs(v, refs) + node := rootNode{ + Title: k, + Description: getDescription(v, item.topLevel), + TopLevel: item.topLevel, + Example: getExample(v), + Type: getHumanReadableType(v.Type), + } + + node.Attributes = getAttributes(v.Properties, refs, customFields, k) + rootProps = append(rootProps, extractNodes(k, v.Properties, refs, customFields)...) + + additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema) + if ok { + objectKeyType := resolveRefs(additionalProps, refs) + d := getDescription(objectKeyType, true) + if d != "" { + node.Description = d + } + if len(node.Example) == 0 { + node.Example = getExample(objectKeyType) + } + node.ObjectKeyAttributes = getAttributes(objectKeyType.Properties, refs, customFields, k) + rootProps = append(rootProps, extractNodes(k, objectKeyType.Properties, refs, customFields)...) + } + + if v.Items != nil { + arrayItemType := resolveRefs(v.Items, refs) + node.ArrayItemAttributes = getAttributes(arrayItemType.Properties, refs, customFields, k) + } + + isEmpty := node.Description == "" && len(node.Attributes) == 0 && len(node.ObjectKeyAttributes) == 0 && len(node.ArrayItemAttributes) == 0 + shouldAddNode := !isEmpty || node.TopLevel + if shouldAddNode { + nodes = append(nodes, node) + } + } + + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].Title < nodes[j].Title + }) + return nodes +} + +func removePluralForm(s string) string { + if strings.HasSuffix(s, "s") { + return strings.TrimSuffix(s, "s") + } + return s +} + +func getHumanReadableType(t jsonschema.Type) string { + typesMapping := map[string]string{ + "string": "String", + "integer": "Integer", + "boolean": "Boolean", + "array": "Sequence", + "object": "Map", + } + return typesMapping[string(t)] +} + +func getAttributes(props map[string]*jsonschema.Schema, refs map[string]jsonschema.Schema, customFields map[string]bool, prefix string) []attributeNode { + attributes := []attributeNode{} + for k, v := range props { + v = resolveRefs(v, refs) + typeString := getHumanReadableType(v.Type) + if typeString == "" { + typeString = "Any" + } + var reference string + if isReferenceType(v, refs, customFields) { + reference = prefix + "." + k + } + attributes = append(attributes, attributeNode{ + Title: k, + Type: typeString, + Description: getDescription(v, true), + Reference: reference, + }) + } + sort.Slice(attributes, func(i, j int) bool { + return attributes[i].Title < attributes[j].Title + }) + return attributes +} + +func getDescription(s *jsonschema.Schema, allowMarkdown bool) string { + if allowMarkdown && s.MarkdownDescription != "" { + return s.MarkdownDescription + } + return s.Description +} + +func shouldExtract(ref string, customFields map[string]bool) bool { + if i := strings.Index(ref, "github.com"); i >= 0 { + ref = ref[i:] + } + _, isCustomField := customFields[ref] + return isCustomField +} + +func extractNodes(prefix string, props map[string]*jsonschema.Schema, refs map[string]jsonschema.Schema, customFields map[string]bool) []rootProp { + nodes := []rootProp{} + for k, v := range props { + if !shouldExtract(*v.Reference, customFields) { + continue + } + v = resolveRefs(v, refs) + if v.Type == "object" || v.Type == "array" { + nodes = append(nodes, rootProp{prefix + "." + k, v, false}) + } + } + return nodes +} + +func getExample(v *jsonschema.Schema) string { + examples := v.Examples + if len(examples) == 0 { + return "" + } + return examples[0].(string) +} diff --git a/bundle/internal/docs/refs.go b/bundle/internal/docs/refs.go new file mode 100644 index 0000000000..11651041b7 --- /dev/null +++ b/bundle/internal/docs/refs.go @@ -0,0 +1,92 @@ +package main + +import ( + "log" + "strings" + + "github.com/databricks/cli/libs/jsonschema" +) + +func isReferenceType(v *jsonschema.Schema, refs map[string]jsonschema.Schema, customFields map[string]bool) bool { + if v.Type != "object" && v.Type != "array" { + return false + } + if len(v.Properties) > 0 { + return true + } + if v.Items != nil { + items := resolveRefs(v.Items, refs) + if items != nil && items.Type == "object" { + return true + } + } + props := resolveAdditionaProperties(v) + if !isInOwnFields(props, customFields) { + return false + } + if props != nil { + propsResolved := resolveRefs(props, refs) + return propsResolved.Type == "object" + } + + return false +} + +func isInOwnFields(node *jsonschema.Schema, customFields map[string]bool) bool { + if node != nil && node.Reference != nil { + return customFields[getRefType(node)] + } + return true +} + +func resolveAdditionaProperties(v *jsonschema.Schema) *jsonschema.Schema { + if v.AdditionalProperties == nil { + return nil + } + additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema) + if !ok { + return nil + } + return additionalProps +} + +func resolveRefs(s *jsonschema.Schema, schemas map[string]jsonschema.Schema) *jsonschema.Schema { + node := s + + description := s.Description + markdownDescription := s.MarkdownDescription + examples := s.Examples + + for node.Reference != nil { + ref := getRefType(node) + newNode, ok := schemas[ref] + if !ok { + log.Printf("schema %s not found", ref) + } + + if description == "" { + description = newNode.Description + } + if markdownDescription == "" { + markdownDescription = newNode.MarkdownDescription + } + if len(examples) == 0 { + examples = newNode.Examples + } + + node = &newNode + } + + node.Description = description + node.MarkdownDescription = markdownDescription + node.Examples = examples + + return node +} + +func getRefType(node *jsonschema.Schema) string { + if node.Reference == nil { + return "" + } + return strings.TrimPrefix(*node.Reference, "#/$defs/") +} diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 91aaa4555f..54631835ec 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -11,6 +11,7 @@ import ( yaml3 "gopkg.in/yaml.v3" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/merge" @@ -19,60 +20,23 @@ import ( "github.com/databricks/cli/libs/jsonschema" ) -type annotation struct { - Description string `json:"description,omitempty"` - MarkdownDescription string `json:"markdown_description,omitempty"` - Title string `json:"title,omitempty"` - Default any `json:"default,omitempty"` - Enum []any `json:"enum,omitempty"` -} - type annotationHandler struct { // Annotations read from all annotation files including all overrides - parsedAnnotations annotationFile + parsedAnnotations annotation.File // Missing annotations for fields that are found in config that need to be added to the annotation file - missingAnnotations annotationFile + missingAnnotations annotation.File } -/** - * Parsed file with annotations, expected format: - * github.com/databricks/cli/bundle/config.Bundle: - * cluster_id: - * description: "Description" - */ -type annotationFile map[string]map[string]annotation - -const Placeholder = "PLACEHOLDER" - // Adds annotations to the JSON schema reading from the annotation files. // More details https://json-schema.org/understanding-json-schema/reference/annotations func newAnnotationHandler(sources []string) (*annotationHandler, error) { - prev := dyn.NilValue - for _, path := range sources { - b, err := os.ReadFile(path) - if err != nil { - return nil, err - } - generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b)) - if err != nil { - return nil, err - } - prev, err = merge.Merge(prev, generated) - if err != nil { - return nil, err - } - } - - var data annotationFile - - err := convert.ToTyped(&data, prev) + data, err := annotation.LoadAndMerge(sources) if err != nil { return nil, err } - d := &annotationHandler{} d.parsedAnnotations = data - d.missingAnnotations = annotationFile{} + d.missingAnnotations = annotation.File{} return d, nil } @@ -85,7 +49,7 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema annotations := d.parsedAnnotations[refPath] if annotations == nil { - annotations = map[string]annotation{} + annotations = map[string]annotation.Descriptor{} } rootTypeAnnotation, ok := annotations[RootTypeKey] @@ -96,11 +60,11 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema for k, v := range s.Properties { item := annotations[k] if item.Description == "" { - item.Description = Placeholder + item.Description = annotation.Placeholder emptyAnnotations := d.missingAnnotations[refPath] if emptyAnnotations == nil { - emptyAnnotations = map[string]annotation{} + emptyAnnotations = map[string]annotation.Descriptor{} d.missingAnnotations[refPath] = emptyAnnotations } emptyAnnotations[k] = item @@ -138,7 +102,7 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error return err } - var outputTyped annotationFile + var outputTyped annotation.File err = convert.ToTyped(&outputTyped, output) if err != nil { return err @@ -155,8 +119,8 @@ func getPath(typ reflect.Type) string { return typ.PkgPath() + "." + typ.Name() } -func assignAnnotation(s *jsonschema.Schema, a annotation) { - if a.Description != Placeholder { +func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { + if a.Description != annotation.Placeholder { s.Description = a.Description } @@ -168,7 +132,7 @@ func assignAnnotation(s *jsonschema.Schema, a annotation) { s.Enum = a.Enum } -func saveYamlWithStyle(outputPath string, annotations annotationFile) error { +func saveYamlWithStyle(outputPath string, annotations annotation.File) error { annotationOrder := yamlsaver.NewOrder([]string{"description", "markdown_description", "title", "default", "enum"}) style := map[string]yaml3.Style{} @@ -220,15 +184,17 @@ func convertLinksToAbsoluteUrl(s string) string { referencePage := "/dev-tools/bundles/reference.html" // Regular expression to match Markdown-style links like [_](link) - re := regexp.MustCompile(`\[_\]\(([^)]+)\)`) + re := regexp.MustCompile(`\[(.*?)\]\((.*?)\)`) result := re.ReplaceAllStringFunc(s, func(match string) string { matches := re.FindStringSubmatch(match) if len(matches) < 2 { return match } - link := matches[1] - var text, absoluteURL string + originalText := matches[1] + link := matches[2] + + var text, absoluteURL string if strings.HasPrefix(link, "#") { text = strings.TrimPrefix(link, "#") absoluteURL = fmt.Sprintf("%s%s%s", base, referencePage, link) @@ -246,6 +212,10 @@ func convertLinksToAbsoluteUrl(s string) string { return match } + if originalText != "_" { + text = originalText + } + return fmt.Sprintf("[%s](%s)", text, absoluteURL) }) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 5283a431be..6b5d873b3f 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1,31 +1,25 @@ github.com/databricks/cli/bundle/config.Artifact: "build": "description": |- - An optional set of non-default build commands that you want to run locally before deployment. - - For Python wheel builds, the Databricks CLI assumes that it can find a local install of the Python wheel package to run builds, and it runs the command python setup.py bdist_wheel by default during each bundle deployment. - - To specify multiple build commands, separate each command with double-ampersand (&&) characters. + An optional set of non-default build commands to run locally before deployment. "executable": "description": |- - The executable type. + The executable type. Valid values are `bash`, `sh`, and `cmd`. "files": "description": |- The source files for the artifact. - "markdown_description": |- - The source files for the artifact, defined as an [_](#artifact_file). "path": "description": |- The location where the built artifact will be saved. "type": "description": |- - The type of the artifact. + Required. The type of the artifact. "markdown_description": |- - The type of the artifact. Valid values are `wheel` or `jar` + Required. The type of the artifact. Valid values are `whl`. github.com/databricks/cli/bundle/config.ArtifactFile: "source": "description": |- - The path of the files used to build the artifact. + Required. The path of the files used to build the artifact. github.com/databricks/cli/bundle/config.Bundle: "cluster_id": "description": |- @@ -44,12 +38,12 @@ github.com/databricks/cli/bundle/config.Bundle: "description": |- The definition of the bundle deployment "markdown_description": |- - The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md). + The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). "git": "description": |- The Git version control details that are associated with your bundle. "markdown_description": |- - The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git). + The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). "name": "description": |- The name of the bundle. @@ -63,8 +57,6 @@ github.com/databricks/cli/bundle/config.Deployment: "lock": "description": |- The deployment lock attributes. - "markdown_description": |- - The deployment lock attributes. See [_](#lock). github.com/databricks/cli/bundle/config.Experimental: "pydabs": "description": |- @@ -74,13 +66,13 @@ github.com/databricks/cli/bundle/config.Experimental: Configures loading of Python code defined with 'databricks-bundles' package. "python_wheel_wrapper": "description": |- - Whether to use a Python wheel wrapper + Whether to use a Python wheel wrapper. "scripts": "description": |- - The commands to run + The commands to run. "use_legacy_run_as": "description": |- - Whether to use the legacy run_as behavior + Whether to use the legacy run_as behavior. github.com/databricks/cli/bundle/config.Git: "branch": "description": |- @@ -149,66 +141,80 @@ github.com/databricks/cli/bundle/config.Python: github.com/databricks/cli/bundle/config.Resources: "clusters": "description": |- - The cluster definitions for the bundle. + The cluster definitions for the bundle, where each key is the name of a cluster. "markdown_description": |- - The cluster definitions for the bundle. See [_](/dev-tools/bundles/resources.md#cluster) + The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#cluster) "dashboards": "description": |- - The dashboard definitions for the bundle. + The dashboard definitions for the bundle, where each key is the name of the dashboard. "markdown_description": |- - The dashboard definitions for the bundle. See [_](/dev-tools/bundles/resources.md#dashboard) + The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboard) "experiments": "description": |- - The experiment definitions for the bundle. + The experiment definitions for the bundle, where each key is the name of the experiment. "markdown_description": |- - The experiment definitions for the bundle. See [_](/dev-tools/bundles/resources.md#experiment) + The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiment) "jobs": "description": |- - The job definitions for the bundle. + The job definitions for the bundle, where each key is the name of the job. "markdown_description": |- - The job definitions for the bundle. See [_](/dev-tools/bundles/resources.md#job) + The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#job) "model_serving_endpoints": "description": |- - The model serving endpoint definitions for the bundle. + The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. "markdown_description": |- - The model serving endpoint definitions for the bundle. See [_](/dev-tools/bundles/resources.md#model_serving_endpoint) + The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoint) "models": "description": |- - The model definitions for the bundle. + The model definitions for the bundle, where each key is the name of the model. "markdown_description": |- - The model definitions for the bundle. See [_](/dev-tools/bundles/resources.md#model) + The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#model-legacy) "pipelines": "description": |- - The pipeline definitions for the bundle. + The pipeline definitions for the bundle, where each key is the name of the pipeline. "markdown_description": |- - The pipeline definitions for the bundle. See [_](/dev-tools/bundles/resources.md#pipeline) + The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipeline) "quality_monitors": "description": |- - The quality monitor definitions for the bundle. + The quality monitor definitions for the bundle, where each key is the name of the quality monitor. "markdown_description": |- - The quality monitor definitions for the bundle. See [_](/dev-tools/bundles/resources.md#quality_monitor) + The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality-monitor) "registered_models": "description": |- - The registered model definitions for the bundle. + The registered model definitions for the bundle, where each key is the name of the registered model. "markdown_description": |- - The registered model definitions for the bundle. See [_](/dev-tools/bundles/resources.md#registered_model) + The registered model definitions for the bundle, where each key is the name of the registered model. See [_](/dev-tools/bundles/resources.md#registered-model) "schemas": "description": |- - The schema definitions for the bundle. + The schema definitions for the bundle, where each key is the name of the schema. "markdown_description": |- - The schema definitions for the bundle. See [_](/dev-tools/bundles/resources.md#schema) + The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schema) "volumes": "description": |- - PLACEHOLDER + The volume definitions for the bundle, where each key is the name of the volume. + "markdown_description": |- + The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volume) github.com/databricks/cli/bundle/config.Root: "artifacts": "description": |- Defines the attributes to build an artifact + "markdown_description": |- + Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [_](/dev-tools/bundles/settings.md#artifacts). + + Artifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [_](/dev-tools/bundles/artifact-overrides.md). + "markdown_examples": |- + ```yaml + artifacts: + default: + type: whl + build: poetry build + path: . + ``` "bundle": "description": |- - The attributes of the bundle. + The bundle attributes when deploying to this target. "markdown_description": |- - The attributes of the bundle. See [_](/dev-tools/bundles/settings.md#bundle) + The bundle attributes when deploying to this target, "experimental": "description": |- Defines attributes for experimental features. @@ -219,9 +225,21 @@ github.com/databricks/cli/bundle/config.Root: Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include) "permissions": "description": |- - Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle + Defines a permission for a specific entity. "markdown_description": |- - Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle. See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). + A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity. + + See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). + "markdown_examples": |- + ```yaml + permissions: + - level: CAN_VIEW + group_name: test-group + - level: CAN_MANAGE + user_name: someone@example.com + - level: CAN_RUN + service_principal_name: 123456-abcdef + ``` "presets": "description": |- Defines bundle deployment presets. @@ -229,26 +247,39 @@ github.com/databricks/cli/bundle/config.Root: Defines bundle deployment presets. See [_](/dev-tools/bundles/deployment-modes.md#presets). "resources": "description": |- - Specifies information about the Databricks resources used by the bundle + A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. "markdown_description": |- - Specifies information about the Databricks resources used by the bundle. See [_](/dev-tools/bundles/resources.md). + A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about supported resources, and resource definition reference, see [_](/dev-tools/bundles/resources.md). + + ```yaml + resources: + s: + : + : + ``` "run_as": "description": |- - The identity to use to run the bundle. + The identity to use when running workflows. + "markdown_description": |- + The identity to use when running workflows. See [_](/dev-tools/bundles/run-as.md). "sync": "description": |- The files and file paths to include or exclude in the bundle. "markdown_description": |- - The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/) + The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/settings.md#sync). "targets": "description": |- Defines deployment targets for the bundle. + "markdown_description": |- + Defines deployment targets for the bundle. See [_](/dev-tools/bundles/settings.md#targets) "variables": "description": |- A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable. "workspace": "description": |- Defines the Databricks workspace for the bundle. + "markdown_description": |- + Defines the Databricks workspace for the bundle. See [_](/dev-tools/bundles/settings.md#workspace). github.com/databricks/cli/bundle/config.Sync: "exclude": "description": |- @@ -263,11 +294,9 @@ github.com/databricks/cli/bundle/config.Target: "artifacts": "description": |- The artifacts to include in the target deployment. - "markdown_description": |- - The artifacts to include in the target deployment. See [_](#artifact) "bundle": "description": |- - The name of the bundle when deploying to this target. + The bundle attributes when deploying to this target. "cluster_id": "description": |- The ID of the cluster to use for this target. @@ -280,8 +309,6 @@ github.com/databricks/cli/bundle/config.Target: "git": "description": |- The Git version control settings for the target. - "markdown_description": |- - The Git version control settings for the target. See [_](#git). "mode": "description": |- The deployment mode for the target. @@ -290,38 +317,26 @@ github.com/databricks/cli/bundle/config.Target: "permissions": "description": |- The permissions for deploying and running the bundle in the target. - "markdown_description": |- - The permissions for deploying and running the bundle in the target. See [_](#permission). "presets": "description": |- The deployment presets for the target. - "markdown_description": |- - The deployment presets for the target. See [_](#preset). "resources": "description": |- The resource definitions for the target. - "markdown_description": |- - The resource definitions for the target. See [_](#resources). "run_as": "description": |- The identity to use to run the bundle. "markdown_description": |- - The identity to use to run the bundle. See [_](#job_run_as) and [_](/dev-tools/bundles/run_as.md). + The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). "sync": "description": |- The local paths to sync to the target workspace when a bundle is run or deployed. - "markdown_description": |- - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#sync). "variables": "description": |- The custom variable definitions for the target. - "markdown_description": |- - The custom variable definitions for the target. See [_](/dev-tools/bundles/settings.md#variables) and [_](/dev-tools/bundles/variables.md). "workspace": "description": |- The Databricks workspace for the target. - "markdown_description": |- - The Databricks workspace for the target. [_](#workspace) github.com/databricks/cli/bundle/config.Workspace: "artifact_path": "description": |- @@ -379,6 +394,11 @@ github.com/databricks/cli/bundle/config/resources.Grant: "description": |- The privileges to grant to the specified entity github.com/databricks/cli/bundle/config/resources.Permission: + "-": + "description": |- + Defines a permission for a specific entity. + "markdown_description": |- + Defines a permission for a specific entity. See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). "group_name": "description": |- The name of the group that has the permission set in level. @@ -445,6 +465,11 @@ github.com/databricks/cli/bundle/config/variable.TargetVariable: "description": |- The type of the variable. github.com/databricks/cli/bundle/config/variable.Variable: + "_": + "description": |- + Defines a custom variable for the bundle. + "markdown_description": |- + Defines a custom variable for the bundle. See [_](/dev-tools/bundles/settings.md#variables). "default": "description": |- PLACEHOLDER @@ -455,7 +480,14 @@ github.com/databricks/cli/bundle/config/variable.Variable: "description": |- The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. "markdown_description": |- - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID." + The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. "type": "description": |- The type of the variable. +github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs: + "service_principal_name": + "description": |- + The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + "user_name": + "description": |- + The email of an active workspace user. Non-admin users can only set this field to their own email. diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index ef602d6efe..99ea27c625 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -1,4 +1,34 @@ github.com/databricks/cli/bundle/config/resources.Cluster: + "_": + "markdown_description": |- + The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). + + "markdown_examples": |- + The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: + + ```yaml + bundle: + name: clusters + + resources: + clusters: + my_cluster: + num_workers: 2 + node_type_id: "i3.xlarge" + autoscale: + min_workers: 2 + max_workers: 7 + spark_version: "13.3.x-scala2.12" + spark_conf: + "spark.executor.memory": "2g" + + jobs: + my_job: + tasks: + - task_key: test_task + notebook_task: + notebook_path: "./src/my_notebook.py" + ``` "data_security_mode": "description": |- PLACEHOLDER @@ -18,6 +48,24 @@ github.com/databricks/cli/bundle/config/resources.Cluster: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Dashboard: + "_": + "markdown_description": |- + The dashboard resource allows you to manage [AI/BI dashboards](/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [_](/dashboards/index.md). + "markdown_examples": |- + The following example includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to the Databricks workspace. + + ``` yaml + resources: + dashboards: + nyc_taxi_trip_analysis: + display_name: "NYC Taxi Trip Analysis" + file_path: ../src/nyc_taxi_trip_analysis.lvdash.json + warehouse_id: ${var.warehouse_id} + ``` + If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). + + In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). + "embed_credentials": "description": |- PLACEHOLDER @@ -28,6 +76,24 @@ github.com/databricks/cli/bundle/config/resources.Dashboard: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Job: + "_": + "markdown_description": |- + The job resource allows you to define [jobs and their corresponding tasks](/api/workspace/jobs/create) in your bundle. For information about jobs, see [_](/jobs/index.md). For a tutorial that uses a template to create a job, see [_](/dev-tools/bundles/jobs-tutorial.md). + "markdown_examples": |- + The following example defines a job with the resource key `hello-job` with one notebook task: + + ```yaml + resources: + jobs: + hello-job: + name: hello-job + tasks: + - task_key: hello-task + notebook_task: + notebook_path: ./hello.py + ``` + + For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md). "health": "description": |- PLACEHOLDER @@ -38,30 +104,186 @@ github.com/databricks/cli/bundle/config/resources.Job: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.MlflowExperiment: + "_": + "markdown_description": |- + The experiment resource allows you to define [MLflow experiments](/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [_](/mlflow/experiments.md). + "markdown_examples": |- + The following example defines an experiment that all users can view: + + ```yaml + resources: + experiments: + experiment: + name: my_ml_experiment + permissions: + - level: CAN_READ + group_name: users + description: MLflow experiment used to track runs + ``` "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.MlflowModel: + "_": + "markdown_description": |- + The model resource allows you to define [legacy models](/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use [registered models](#registered-model) instead. "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: + "_": + "markdown_description": |- + The model_serving_endpoint resource allows you to define [model serving endpoints](/api/workspace/servingendpoints/create). See [_](/machine-learning/model-serving/manage-serving-endpoints.md). + "markdown_examples": |- + The following example defines a model serving endpoint: + + ```yaml + resources: + model_serving_endpoints: + uc_model_serving_endpoint: + name: "uc-model-endpoint" + config: + served_entities: + - entity_name: "myCatalog.mySchema.my-ads-model" + entity_version: "10" + workload_size: "Small" + scale_to_zero_enabled: "true" + traffic_config: + routes: + - served_model_name: "my-ads-model-10" + traffic_percentage: "100" + tags: + - key: "team" + value: "data science" + ``` "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Pipeline: + "_": + "markdown_description": |- + The pipeline resource allows you to create [pipelines](/api/workspace/pipelines/create). For information about pipelines, see [_](/delta-live-tables/index.md). For a tutorial that uses the template to create a pipeline, see [_](/dev-tools/bundles/pipelines-tutorial.md). + "markdown_examples": |- + The following example defines a pipeline with the resource key `hello-pipeline`: + + ```yaml + resources: + pipelines: + hello-pipeline: + name: hello-pipeline + clusters: + - label: default + num_workers: 1 + development: true + continuous: false + channel: CURRENT + edition: CORE + photon: false + libraries: + - notebook: + path: ./pipeline.py + ``` "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.QualityMonitor: + "_": + "markdown_description": |- + The quality_monitor resource allows you to define a [table monitor](/api/workspace/qualitymonitors/create). For information about monitors, see [_](/machine-learning/model-serving/monitor-diagnose-endpoints.md). + "markdown_examples": |- + The following example defines a quality monitor: + + ```yaml + resources: + quality_monitors: + my_quality_monitor: + table_name: dev.mlops_schema.predictions + output_schema_name: ${bundle.target}.mlops_schema + assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + label_col: price + problem_type: PROBLEM_TYPE_REGRESSION + timestamp_col: timestamp + schedule: + quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am + timezone_id: UTC + ``` "table_name": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.RegisteredModel: + "_": + "markdown_description": |- + The registered model resource allows you to define models in . For information about [registered models](/api/workspace/registeredmodels/create), see [_](/machine-learning/manage-model-lifecycle/index.md). + "markdown_examples": |- + The following example defines a registered model in : + + ```yaml + resources: + registered_models: + model: + name: my_model + catalog_name: ${bundle.target} + schema_name: mlops_schema + comment: Registered model in Unity Catalog for ${bundle.target} deployment target + grants: + - privileges: + - EXECUTE + principal: account users + ``` "grants": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Schema: + "_": + "markdown_description": |- + The schema resource type allows you to define [schemas](/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations: + + - The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema. + - Only fields supported by the corresponding [Schemas object create API](/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](/api/workspace/schemas/update). + "markdown_examples": |- + The following example defines a pipeline with the resource key `my_pipeline` that creates a schema with the key `my_schema` as the target: + + ```yaml + resources: + pipelines: + my_pipeline: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: main + target: ${resources.schemas.my_schema.id} + + schemas: + my_schema: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created by DABs. + ``` + + A top-level grants mapping is not supported by , so if you want to set grants for a schema, define the grants for the schema within the `schemas` mapping. For more information about grants, see [_](/data-governance/unity-catalog/manage-privileges/index.md#grant). + + The following example defines a schema with grants: + + ```yaml + resources: + schemas: + my_schema: + name: test-schema + grants: + - principal: users + privileges: + - CAN_MANAGE + - principal: my_team + privileges: + - CAN_READ + catalog_name: main + ``` "grants": "description": |- PLACEHOLDER @@ -69,6 +291,27 @@ github.com/databricks/cli/bundle/config/resources.Schema: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Volume: + "_": + "markdown_description": |- + The volume resource type allows you to define and create [volumes](/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that: + + - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. + + - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). + + "markdown_examples": |- + The following example creates a volume with the key `my_volume`: + + ```yaml + resources: + volumes: + my_volume: + catalog_name: main + name: my_volume + schema_name: my_schema + ``` + + For an example bundle that runs a job that writes to a file in volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume). "grants": "description": |- PLACEHOLDER diff --git a/bundle/internal/schema/annotations_test.go b/bundle/internal/schema/annotations_test.go index d7e2fea7cf..0e15933596 100644 --- a/bundle/internal/schema/annotations_test.go +++ b/bundle/internal/schema/annotations_test.go @@ -33,6 +33,10 @@ func TestConvertLinksToAbsoluteUrl(t *testing.T) { input: "This is a link to [external](https://external.com)", expected: "This is a link to [external](https://external.com)", }, + { + input: "This is a link to [one](/relative), [two](/relative-2)", + expected: "This is a link to [one](https://docs.databricks.com/relative), [two](https://docs.databricks.com/relative-2)", + }, } for _, test := range tests { diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 06e89c856f..051243c4d7 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/dyn/yamlloader" @@ -113,13 +114,13 @@ func TestNoDetachedAnnotations(t *testing.T) { assert.Empty(t, types, "Detached annotations found, regenerate schema and check for package path changes") } -func getAnnotations(path string) (annotationFile, error) { +func getAnnotations(path string) (annotation.File, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } - var data annotationFile + var data annotation.File err = yaml.Unmarshal(b, &data) return data, err } diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 9199084299..50e69e7c83 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/jsonschema" "gopkg.in/yaml.v3" ) @@ -114,8 +115,8 @@ func mapIncorrectTypNames(ref string) string { // Use the OpenAPI spec to load descriptions for the given type. func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overridesPath string) error { - annotations := annotationFile{} - overrides := annotationFile{} + annotations := annotation.File{} + overrides := annotation.File{} b, err := os.ReadFile(overridesPath) if err != nil { @@ -126,7 +127,7 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid return err } if overrides == nil { - overrides = annotationFile{} + overrides = annotation.File{} } _, err = jsonschema.FromType(typ, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ @@ -137,16 +138,16 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid } basePath := getPath(typ) - pkg := map[string]annotation{} + pkg := map[string]annotation.Descriptor{} annotations[basePath] = pkg if ref.Description != "" || ref.Enum != nil { - pkg[RootTypeKey] = annotation{Description: ref.Description, Enum: ref.Enum} + pkg[RootTypeKey] = annotation.Descriptor{Description: ref.Description, Enum: ref.Enum} } for k := range s.Properties { if refProp, ok := ref.Properties[k]; ok { - pkg[k] = annotation{Description: refProp.Description, Enum: refProp.Enum} + pkg[k] = annotation.Descriptor{Description: refProp.Description, Enum: refProp.Enum} if refProp.Description == "" { addEmptyOverride(k, basePath, overrides) } @@ -195,22 +196,22 @@ func prependCommentToFile(outputPath, comment string) error { return err } -func addEmptyOverride(key, pkg string, overridesFile annotationFile) { +func addEmptyOverride(key, pkg string, overridesFile annotation.File) { if overridesFile[pkg] == nil { - overridesFile[pkg] = map[string]annotation{} + overridesFile[pkg] = map[string]annotation.Descriptor{} } overrides := overridesFile[pkg] if overrides[key].Description == "" { - overrides[key] = annotation{Description: Placeholder} + overrides[key] = annotation.Descriptor{Description: annotation.Placeholder} } a, ok := overrides[key] if !ok { - a = annotation{} + a = annotation.Descriptor{} } if a.Description == "" { - a.Description = Placeholder + a.Description = annotation.Placeholder } overrides[key] = a } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2f78ffcca2..50f1c6f956 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -183,7 +183,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The cluster resource defines an [all-purpose cluster](https://docs.databricks.com/api/workspace/clusters/create)." }, { "type": "string", @@ -246,7 +247,8 @@ "$ref": "#/$defs/string" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The dashboard resource allows you to manage [AI/BI dashboards](https://docs.databricks.com/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [link](https://docs.databricks.com/dashboards/index.html)." }, { "type": "string", @@ -367,7 +369,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The job resource allows you to define [jobs and their corresponding tasks](https://docs.databricks.com/api/workspace/jobs/create) in your bundle. For information about jobs, see [link](https://docs.databricks.com/jobs/index.html). For a tutorial that uses a \u003cDABS\u003e template to create a job, see [link](https://docs.databricks.com/dev-tools/bundles/jobs-tutorial.html)." }, { "type": "string", @@ -412,7 +415,8 @@ "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The experiment resource allows you to define [MLflow experiments](https://docs.databricks.com/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [link](https://docs.databricks.com/mlflow/experiments.html)." }, { "type": "string", @@ -457,7 +461,8 @@ "$ref": "#/$defs/string" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The model resource allows you to define [legacy models](https://docs.databricks.com/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use \u003cUC\u003e [registered models](https://docs.databricks.com/dev-tools/bundles/reference.html#registered-model) instead." }, { "type": "string", @@ -502,7 +507,8 @@ "required": [ "config", "name" - ] + ], + "markdownDescription": "The model_serving_endpoint resource allows you to define [model serving endpoints](https://docs.databricks.com/api/workspace/servingendpoints/create). See [link](https://docs.databricks.com/machine-learning/model-serving/manage-serving-endpoints.html)." }, { "type": "string", @@ -644,7 +650,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The pipeline resource allows you to create \u003cDLT\u003e [pipelines](https://docs.databricks.com/api/workspace/pipelines/create). For information about pipelines, see [link](https://docs.databricks.com/delta-live-tables/index.html). For a tutorial that uses the \u003cDABS\u003e template to create a pipeline, see [link](https://docs.databricks.com/dev-tools/bundles/pipelines-tutorial.html)." }, { "type": "string", @@ -718,7 +725,8 @@ "table_name", "assets_dir", "output_schema_name" - ] + ], + "markdownDescription": "The quality_monitor resource allows you to define a \u003cUC\u003e [table monitor](https://docs.databricks.com/api/workspace/qualitymonitors/create). For information about monitors, see [link](https://docs.databricks.com/machine-learning/model-serving/monitor-diagnose-endpoints.html)." }, { "type": "string", @@ -760,7 +768,8 @@ "catalog_name", "name", "schema_name" - ] + ], + "markdownDescription": "The registered model resource allows you to define models in \u003cUC\u003e. For information about \u003cUC\u003e [registered models](https://docs.databricks.com/api/workspace/registeredmodels/create), see [link](https://docs.databricks.com/machine-learning/manage-model-lifecycle/index.html)." }, { "type": "string", @@ -800,7 +809,8 @@ "required": [ "catalog_name", "name" - ] + ], + "markdownDescription": "The schema resource type allows you to define \u003cUC\u003e [schemas](https://docs.databricks.com/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations:\n\n- The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema.\n- Only fields supported by the corresponding [Schemas object create API](https://docs.databricks.com/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](https://docs.databricks.com/api/workspace/schemas/update)." }, { "type": "string", @@ -845,7 +855,8 @@ "catalog_name", "name", "schema_name" - ] + ], + "markdownDescription": "The volume resource type allows you to define and create \u003cUC\u003e [volumes](https://docs.databricks.com/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that:\n\n- A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use \u003cDABS\u003e to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.\n\n- Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [custom-presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#custom-presets)." }, { "type": "string", @@ -931,6 +942,7 @@ }, "variable.Variable": { "type": "object", + "description": "Defines a custom variable for the bundle.", "properties": { "default": { "$ref": "#/$defs/interface" @@ -949,7 +961,8 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "Defines a custom variable for the bundle. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables)." }, "variable.VariableType": { "type": "string" @@ -961,26 +974,25 @@ "type": "object", "properties": { "build": { - "description": "An optional set of non-default build commands that you want to run locally before deployment.\n\nFor Python wheel builds, the Databricks CLI assumes that it can find a local install of the Python wheel package to run builds, and it runs the command python setup.py bdist_wheel by default during each bundle deployment.\n\nTo specify multiple build commands, separate each command with double-ampersand (\u0026\u0026) characters.", + "description": "An optional set of non-default build commands to run locally before deployment.", "$ref": "#/$defs/string" }, "executable": { - "description": "The executable type.", + "description": "The executable type. Valid values are `bash`, `sh`, and `cmd`.", "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType" }, "files": { "description": "The source files for the artifact.", - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile", - "markdownDescription": "The source files for the artifact, defined as an [artifact_file](https://docs.databricks.com/dev-tools/bundles/reference.html#artifact_file)." + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile" }, "path": { "description": "The location where the built artifact will be saved.", "$ref": "#/$defs/string" }, "type": { - "description": "The type of the artifact.", + "description": "Required. The type of the artifact.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType", - "markdownDescription": "The type of the artifact. Valid values are `wheel` or `jar`" + "markdownDescription": "Required. The type of the artifact. Valid values are `whl`." } }, "additionalProperties": false, @@ -1000,7 +1012,7 @@ "type": "object", "properties": { "source": { - "description": "The path of the files used to build the artifact.", + "description": "Required. The path of the files used to build the artifact.", "$ref": "#/$defs/string" } }, @@ -1039,12 +1051,12 @@ "deployment": { "description": "The definition of the bundle deployment", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Deployment", - "markdownDescription": "The definition of the bundle deployment. For supported attributes, see [deployment](https://docs.databricks.com/dev-tools/bundles/reference.html#deployment) and [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)." + "markdownDescription": "The definition of the bundle deployment. For supported attributes see [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)." }, "git": { "description": "The Git version control details that are associated with your bundle.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git", - "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes, see [git](https://docs.databricks.com/dev-tools/bundles/reference.html#git) and [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." + "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes see [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." }, "name": { "description": "The name of the bundle.", @@ -1079,8 +1091,7 @@ }, "lock": { "description": "The deployment lock attributes.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock", - "markdownDescription": "The deployment lock attributes. See [lock](https://docs.databricks.com/dev-tools/bundles/reference.html#lock)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" } }, "additionalProperties": false @@ -1105,15 +1116,15 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python" }, "python_wheel_wrapper": { - "description": "Whether to use a Python wheel wrapper", + "description": "Whether to use a Python wheel wrapper.", "$ref": "#/$defs/bool" }, "scripts": { - "description": "The commands to run", + "description": "The commands to run.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command" }, "use_legacy_run_as": { - "description": "Whether to use the legacy run_as behavior", + "description": "Whether to use the legacy run_as behavior.", "$ref": "#/$defs/bool" } }, @@ -1274,57 +1285,59 @@ "type": "object", "properties": { "clusters": { - "description": "The cluster definitions for the bundle.", + "description": "The cluster definitions for the bundle, where each key is the name of a cluster.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster", - "markdownDescription": "The cluster definitions for the bundle. See [cluster](https://docs.databricks.com/dev-tools/bundles/resources.html#cluster)" + "markdownDescription": "The cluster definitions for the bundle, where each key is the name of a cluster. See [cluster](https://docs.databricks.com/dev-tools/bundles/resources.html#cluster)" }, "dashboards": { - "description": "The dashboard definitions for the bundle.", + "description": "The dashboard definitions for the bundle, where each key is the name of the dashboard.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard", - "markdownDescription": "The dashboard definitions for the bundle. See [dashboard](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboard)" + "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboard](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboard)" }, "experiments": { - "description": "The experiment definitions for the bundle.", + "description": "The experiment definitions for the bundle, where each key is the name of the experiment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment", - "markdownDescription": "The experiment definitions for the bundle. See [experiment](https://docs.databricks.com/dev-tools/bundles/resources.html#experiment)" + "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiment](https://docs.databricks.com/dev-tools/bundles/resources.html#experiment)" }, "jobs": { - "description": "The job definitions for the bundle.", + "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", - "markdownDescription": "The job definitions for the bundle. See [job](https://docs.databricks.com/dev-tools/bundles/resources.html#job)" + "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [job](https://docs.databricks.com/dev-tools/bundles/resources.html#job)" }, "model_serving_endpoints": { - "description": "The model serving endpoint definitions for the bundle.", + "description": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint", - "markdownDescription": "The model serving endpoint definitions for the bundle. See [model_serving_endpoint](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoint)" + "markdownDescription": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [model_serving_endpoint](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoint)" }, "models": { - "description": "The model definitions for the bundle.", + "description": "The model definitions for the bundle, where each key is the name of the model.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel", - "markdownDescription": "The model definitions for the bundle. See [model](https://docs.databricks.com/dev-tools/bundles/resources.html#model)" + "markdownDescription": "The model definitions for the bundle, where each key is the name of the model. See [model-legacy](https://docs.databricks.com/dev-tools/bundles/resources.html#model-legacy)" }, "pipelines": { - "description": "The pipeline definitions for the bundle.", + "description": "The pipeline definitions for the bundle, where each key is the name of the pipeline.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline", - "markdownDescription": "The pipeline definitions for the bundle. See [pipeline](https://docs.databricks.com/dev-tools/bundles/resources.html#pipeline)" + "markdownDescription": "The pipeline definitions for the bundle, where each key is the name of the pipeline. See [pipeline](https://docs.databricks.com/dev-tools/bundles/resources.html#pipeline)" }, "quality_monitors": { - "description": "The quality monitor definitions for the bundle.", + "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor", - "markdownDescription": "The quality monitor definitions for the bundle. See [quality_monitor](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitor)" + "markdownDescription": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [quality-monitor](https://docs.databricks.com/dev-tools/bundles/resources.html#quality-monitor)" }, "registered_models": { - "description": "The registered model definitions for the bundle.", + "description": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel", - "markdownDescription": "The registered model definitions for the bundle. See [registered_model](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_model)" + "markdownDescription": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model. See [registered-model](https://docs.databricks.com/dev-tools/bundles/resources.html#registered-model)" }, "schemas": { - "description": "The schema definitions for the bundle.", + "description": "The schema definitions for the bundle, where each key is the name of the schema.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema", - "markdownDescription": "The schema definitions for the bundle. See [schema](https://docs.databricks.com/dev-tools/bundles/resources.html#schema)" + "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schema](https://docs.databricks.com/dev-tools/bundles/resources.html#schema)" }, "volumes": { - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume" + "description": "The volume definitions for the bundle, where each key is the name of the volume.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", + "markdownDescription": "The volume definitions for the bundle, where each key is the name of the volume. See [volume](https://docs.databricks.com/dev-tools/bundles/resources.html#volume)" } }, "additionalProperties": false @@ -1368,11 +1381,10 @@ "properties": { "artifacts": { "description": "The artifacts to include in the target deployment.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact", - "markdownDescription": "The artifacts to include in the target deployment. See [artifact](https://docs.databricks.com/dev-tools/bundles/reference.html#artifact)" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" }, "bundle": { - "description": "The name of the bundle when deploying to this target.", + "description": "The bundle attributes when deploying to this target.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" }, "cluster_id": { @@ -1389,8 +1401,7 @@ }, "git": { "description": "The Git version control settings for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git", - "markdownDescription": "The Git version control settings for the target. See [git](https://docs.databricks.com/dev-tools/bundles/reference.html#git)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" }, "mode": { "description": "The deployment mode for the target.", @@ -1399,38 +1410,32 @@ }, "permissions": { "description": "The permissions for deploying and running the bundle in the target.", - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission", - "markdownDescription": "The permissions for deploying and running the bundle in the target. See [permission](https://docs.databricks.com/dev-tools/bundles/reference.html#permission)." + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, "presets": { "description": "The deployment presets for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets", - "markdownDescription": "The deployment presets for the target. See [preset](https://docs.databricks.com/dev-tools/bundles/reference.html#preset)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" }, "resources": { "description": "The resource definitions for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources", - "markdownDescription": "The resource definitions for the target. See [resources](https://docs.databricks.com/dev-tools/bundles/reference.html#resources)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" }, "run_as": { "description": "The identity to use to run the bundle.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs", - "markdownDescription": "The identity to use to run the bundle. See [job_run_as](https://docs.databricks.com/dev-tools/bundles/reference.html#job_run_as) and [link](https://docs.databricks.com/dev-tools/bundles/run_as.html)." + "markdownDescription": "The identity to use to run the bundle, see [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)." }, "sync": { "description": "The local paths to sync to the target workspace when a bundle is run or deployed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync", - "markdownDescription": "The local paths to sync to the target workspace when a bundle is run or deployed. See [sync](https://docs.databricks.com/dev-tools/bundles/reference.html#sync)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" }, "variables": { "description": "The custom variable definitions for the target.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable", - "markdownDescription": "The custom variable definitions for the target. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables) and [link](https://docs.databricks.com/dev-tools/bundles/variables.html)." + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable" }, "workspace": { "description": "The Databricks workspace for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace", - "markdownDescription": "The Databricks workspace for the target. [workspace](https://docs.databricks.com/dev-tools/bundles/reference.html#workspace)" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" } }, "additionalProperties": false @@ -3102,7 +3107,7 @@ "description": "Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nEither `user_name` or `service_principal_name` should be specified. If not, an error is thrown.", "properties": { "service_principal_name": { - "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "description": "The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", "$ref": "#/$defs/string" }, "user_name": { @@ -6406,12 +6411,13 @@ "properties": { "artifacts": { "description": "Defines the attributes to build an artifact", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact", + "markdownDescription": "Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [artifacts](https://docs.databricks.com/dev-tools/bundles/settings.html#artifacts).\n\nArtifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [link](https://docs.databricks.com/dev-tools/bundles/artifact-overrides.html)." }, "bundle": { - "description": "The attributes of the bundle.", + "description": "The bundle attributes when deploying to this target.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle", - "markdownDescription": "The attributes of the bundle. See [bundle](https://docs.databricks.com/dev-tools/bundles/settings.html#bundle)" + "markdownDescription": "The bundle attributes when deploying to this target," }, "experimental": { "description": "Defines attributes for experimental features.", @@ -6423,9 +6429,9 @@ "markdownDescription": "Specifies a list of path globs that contain configuration files to include within the bundle. See [include](https://docs.databricks.com/dev-tools/bundles/settings.html#include)" }, "permissions": { - "description": "Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle", + "description": "Defines a permission for a specific entity.", "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission", - "markdownDescription": "Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle. See [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." + "markdownDescription": "A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." }, "presets": { "description": "Defines bundle deployment presets.", @@ -6433,22 +6439,24 @@ "markdownDescription": "Defines bundle deployment presets. See [presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#presets)." }, "resources": { - "description": "Specifies information about the Databricks resources used by the bundle", + "description": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources", - "markdownDescription": "Specifies information about the Databricks resources used by the bundle. See [link](https://docs.databricks.com/dev-tools/bundles/resources.html)." + "markdownDescription": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about \u003cDABS\u003e supported resources, and resource definition reference, see [link](https://docs.databricks.com/dev-tools/bundles/resources.html).\n\n```yaml\nresources:\n \u003cresource-type\u003es:\n \u003cresource-name\u003e:\n \u003cresource-field-name\u003e: \u003cresource-field-value\u003e\n```" }, "run_as": { - "description": "The identity to use to run the bundle.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + "description": "The identity to use when running \u003cDABS\u003e workflows.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs", + "markdownDescription": "The identity to use when running \u003cDABS\u003e workflows. See [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)." }, "sync": { "description": "The files and file paths to include or exclude in the bundle.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync", - "markdownDescription": "The files and file paths to include or exclude in the bundle. See [link](https://docs.databricks.com/dev-tools/bundles/)" + "markdownDescription": "The files and file paths to include or exclude in the bundle. See [sync](https://docs.databricks.com/dev-tools/bundles/settings.html#sync)." }, "targets": { "description": "Defines deployment targets for the bundle.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target", + "markdownDescription": "Defines deployment targets for the bundle. See [targets](https://docs.databricks.com/dev-tools/bundles/settings.html#targets)" }, "variables": { "description": "A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable.", @@ -6456,7 +6464,8 @@ }, "workspace": { "description": "Defines the Databricks workspace for the bundle.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace", + "markdownDescription": "Defines the Databricks workspace for the bundle. See [workspace](https://docs.databricks.com/dev-tools/bundles/settings.html#workspace)." } }, "additionalProperties": false diff --git a/go.mod b/go.mod index 86bc1c368e..70ea1ffffa 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require github.com/nao1215/markdown v0.6.0 + require ( cloud.google.com/go/auth v0.4.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect @@ -54,7 +56,10 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/karrick/godirwalk v1.17.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tidwall/gjson v1.18.0 // indirect diff --git a/go.sum b/go.sum index f6cf796074..1e84331cd5 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= @@ -127,8 +129,14 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/nao1215/markdown v0.6.0 h1:kqhrC47K434YA1jMTUwJwSV/hla8ifN3NzehMEffI/E= +github.com/nao1215/markdown v0.6.0/go.mod h1:ObBhnNduWwPN+bu4dtv4JoLRt57ONla7l//03iHIVhY= github.com/nwidger/jsoncolor v0.3.2 h1:rVJJlwAWDJShnbTYOQ5RM7yTA20INyKXlJ/fg4JMhHQ= github.com/nwidger/jsoncolor v0.3.2/go.mod h1:Cs34umxLbJvgBMnVNVqhji9BhoT/N/KinHqZptQ7cf4= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 6f8f39d961..ce25cb0234 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -111,6 +111,10 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) ( return res, nil } +func TypePath(typ reflect.Type) string { + return typePath(typ) +} + // typePath computes a unique string representation of the type. $ref in the generated // JSON schema will refer to this path. See TestTypePath for examples outputs. func typePath(typ reflect.Type) string { diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index e63dde359e..5028bb0d7b 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -76,6 +76,10 @@ type Schema struct { // Title of the object, rendered as inline documentation in the IDE. // https://json-schema.org/understanding-json-schema/reference/annotations Title string `json:"title,omitempty"` + + // Examples of the value for properties in the schema. + // https://json-schema.org/understanding-json-schema/reference/annotations + Examples []any `json:"examples,omitempty"` } // Default value defined in a JSON Schema, represented as a string.