diff --git a/docs/overlays/elasticsearch-openapi-overlays.yaml b/docs/overlays/elasticsearch-openapi-overlays.yaml index 2361374fd1..629a2a8cb0 100644 --- a/docs/overlays/elasticsearch-openapi-overlays.yaml +++ b/docs/overlays/elasticsearch-openapi-overlays.yaml @@ -394,7 +394,6 @@ actions: examples: indicesLegacyPutTemplateRequestExample1: $ref: "../../specification/indices/put_template/indicesPutTemplateRequestExample1.yaml" - ## Examples for behavioral analytics - target: "$.paths['/_application/analytics/{collection_name}/event/{event_type}']['post']" description: "Add examples for post analytics collection event operation" @@ -416,6 +415,33 @@ actions: examples: dataStreamLifecycleStatsResponseExample1: $ref: "../../specification/indices/get_data_lifecycle_stats/examples/response/IndicesGetDataLifecycleStatsResponseExample1.yaml" +## Examples for ingest + - target: "$.components['requestBodies']['simulate.ingest']" + description: "Add example for simulate ingest request" + update: + content: + application/json: + examples: + simulateIngestRequestExample1: + $ref: "../../specification/simulate/ingest/examples/request/SimulateIngestRequestExample1.yaml" + simulateIngestRequestExample2: + $ref: "../../specification/simulate/ingest/examples/request/SimulateIngestRequestExample2.yaml" + simulateIngestRequestExample3: + $ref: "../../specification/simulate/ingest/examples/request/SimulateIngestRequestExample3.yaml" + simulateIngestRequestExample4: + $ref: "../../specification/simulate/ingest/examples/request/SimulateIngestRequestExample4.yaml" + - target: "$.components['responses']['simulate.ingest#200']" + description: "Add example for simulate ingest response" + update: + content: + application/json: + examples: + simulateIngestResponseExample1: + $ref: "../../specification/simulate/ingest/examples/response/SimulateIngestResponseExample1.yaml" + simulateIngestResponseExample2: + $ref: "../../specification/simulate/ingest/examples/response/SimulateIngestResponseExample2.yaml" + simulateIngestResponseExample3: + $ref: "../../specification/simulate/ingest/examples/response/SimulateIngestResponseExample3.yaml" ## Examples for licensing - target: "$.paths['/_license/basic_status']['get']" description: "Add example for get basic status response" @@ -485,7 +511,7 @@ actions: $ref: "../../specification/search_application/render_query/SearchApplicationsRenderQueryRequestExample1.yaml" renderSearchApplicationQueryResponseExample1: $ref: "../../specification/search_application/render_query/SearchApplicationsRenderQueryResponseExample1.yaml" - +# Examples for security - target: "$.paths['/_security/api_key/_bulk_update']['post']" description: "Add examples for bulk update API keys operation" update: @@ -540,4 +566,4 @@ actions: examples: updateWatcherSettingsRequestExample1: $ref: "../../specification/watcher/get_settings/examples/200_response/WatcherGetSettingsResponseExample1.yaml" - + \ No newline at end of file diff --git a/docs/overlays/elasticsearch-shared-overlays.yaml b/docs/overlays/elasticsearch-shared-overlays.yaml index 8c125fa223..66a093114a 100644 --- a/docs/overlays/elasticsearch-shared-overlays.yaml +++ b/docs/overlays/elasticsearch-shared-overlays.yaml @@ -1292,7 +1292,7 @@ actions: application/json: examples: simulatePipelineRequestExample1: - $ref: "../../specification/ingest/simulate/SimulatePipelineRequestExample1.yaml" + $ref: "../../specification/ingest/simulate/examples/request/SimulatePipelineRequestExample1.yaml" - target: "$.components['responses']['ingest.simulate#200']" description: "Add example for simulate pipeline response" update: @@ -1300,7 +1300,7 @@ actions: application/json: examples: simulatePipelineResponseExample1: - $ref: "../../specification/ingest/simulate/SimulatePipelineResponseExample1.yaml" + $ref: "../../specification/ingest/simulate/examples/response/SimulatePipelineResponseExample1.yaml" - target: "$.components['requestBodies']['indices.put_index_template']" description: "Add example for create index template request" update: diff --git a/output/openapi/elasticsearch-openapi.json b/output/openapi/elasticsearch-openapi.json index 967cb3f2c7..358914b624 100644 --- a/output/openapi/elasticsearch-openapi.json +++ b/output/openapi/elasticsearch-openapi.json @@ -32430,6 +32430,104 @@ "x-state": "Added in 7.13.0" } }, + "/_ingest/_simulate": { + "get": { + "tags": [ + "ingest" + ], + "summary": "Simulate data ingestion", + "description": "Run ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.\n\nThis API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.\n\nThe API runs the default and final pipeline for that index against a set of documents provided in the body of the request.\nIf a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would.\nNo data is indexed into Elasticsearch.\nInstead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation.\nThe transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.\n\nThis API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline.\nThe simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.\n\nBy default, the pipeline definitions that are currently in the system are used.\nHowever, you can supply substitute pipeline definitions in the body of the request.\nThese will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.", + "operationId": "simulate-ingest", + "parameters": [ + { + "$ref": "#/components/parameters/simulate.ingest#pipeline" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/simulate.ingest" + }, + "responses": { + "200": { + "$ref": "#/components/responses/simulate.ingest#200" + } + }, + "x-state": "Technical preview" + }, + "post": { + "tags": [ + "ingest" + ], + "summary": "Simulate data ingestion", + "description": "Run ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.\n\nThis API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.\n\nThe API runs the default and final pipeline for that index against a set of documents provided in the body of the request.\nIf a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would.\nNo data is indexed into Elasticsearch.\nInstead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation.\nThe transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.\n\nThis API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline.\nThe simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.\n\nBy default, the pipeline definitions that are currently in the system are used.\nHowever, you can supply substitute pipeline definitions in the body of the request.\nThese will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.", + "operationId": "simulate-ingest-1", + "parameters": [ + { + "$ref": "#/components/parameters/simulate.ingest#pipeline" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/simulate.ingest" + }, + "responses": { + "200": { + "$ref": "#/components/responses/simulate.ingest#200" + } + }, + "x-state": "Technical preview" + } + }, + "/_ingest/{index}/_simulate": { + "get": { + "tags": [ + "ingest" + ], + "summary": "Simulate data ingestion", + "description": "Run ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.\n\nThis API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.\n\nThe API runs the default and final pipeline for that index against a set of documents provided in the body of the request.\nIf a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would.\nNo data is indexed into Elasticsearch.\nInstead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation.\nThe transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.\n\nThis API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline.\nThe simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.\n\nBy default, the pipeline definitions that are currently in the system are used.\nHowever, you can supply substitute pipeline definitions in the body of the request.\nThese will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.", + "operationId": "simulate-ingest-2", + "parameters": [ + { + "$ref": "#/components/parameters/simulate.ingest#index" + }, + { + "$ref": "#/components/parameters/simulate.ingest#pipeline" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/simulate.ingest" + }, + "responses": { + "200": { + "$ref": "#/components/responses/simulate.ingest#200" + } + }, + "x-state": "Technical preview" + }, + "post": { + "tags": [ + "ingest" + ], + "summary": "Simulate data ingestion", + "description": "Run ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.\n\nThis API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.\n\nThe API runs the default and final pipeline for that index against a set of documents provided in the body of the request.\nIf a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would.\nNo data is indexed into Elasticsearch.\nInstead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation.\nThe transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.\n\nThis API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline.\nThe simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.\n\nBy default, the pipeline definitions that are currently in the system are used.\nHowever, you can supply substitute pipeline definitions in the body of the request.\nThese will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.", + "operationId": "simulate-ingest-3", + "parameters": [ + { + "$ref": "#/components/parameters/simulate.ingest#index" + }, + { + "$ref": "#/components/parameters/simulate.ingest#pipeline" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/simulate.ingest" + }, + "responses": { + "200": { + "$ref": "#/components/responses/simulate.ingest#200" + } + }, + "x-state": "Technical preview" + } + }, "/_slm/policy/{policy_id}": { "get": { "tags": [ @@ -74069,7 +74167,7 @@ "version" ] }, - "ingest.simulate:Document": { + "ingest._types:Document": { "type": "object", "properties": { "_id": { @@ -74087,11 +74185,11 @@ "_source" ] }, - "ingest.simulate:SimulateDocumentResult": { + "ingest._types:SimulateDocumentResult": { "type": "object", "properties": { "doc": { - "$ref": "#/components/schemas/ingest.simulate:DocumentSimulation" + "$ref": "#/components/schemas/ingest._types:DocumentSimulation" }, "error": { "$ref": "#/components/schemas/_types:ErrorCause" @@ -74099,12 +74197,12 @@ "processor_results": { "type": "array", "items": { - "$ref": "#/components/schemas/ingest.simulate:PipelineSimulation" + "$ref": "#/components/schemas/ingest._types:PipelineSimulation" } } } }, - "ingest.simulate:DocumentSimulation": { + "ingest._types:DocumentSimulation": { "type": "object", "properties": { "_id": { @@ -74114,7 +74212,7 @@ "$ref": "#/components/schemas/_types:IndexName" }, "_ingest": { - "$ref": "#/components/schemas/ingest.simulate:Ingest" + "$ref": "#/components/schemas/ingest._types:Ingest" }, "_routing": { "description": "Value used to send the document to a specific primary shard.", @@ -74141,11 +74239,11 @@ "_source" ] }, - "ingest.simulate:Ingest": { + "ingest._types:Ingest": { "type": "object", "properties": { "_redact": { - "$ref": "#/components/schemas/ingest.simulate:Redact" + "$ref": "#/components/schemas/ingest._types:Redact" }, "timestamp": { "$ref": "#/components/schemas/_types:DateTime" @@ -74158,7 +74256,7 @@ "timestamp" ] }, - "ingest.simulate:Redact": { + "ingest._types:Redact": { "type": "object", "properties": { "_is_redacted": { @@ -74181,11 +74279,11 @@ } ] }, - "ingest.simulate:PipelineSimulation": { + "ingest._types:PipelineSimulation": { "type": "object", "properties": { "doc": { - "$ref": "#/components/schemas/ingest.simulate:DocumentSimulation" + "$ref": "#/components/schemas/ingest._types:DocumentSimulation" }, "tag": { "type": "string" @@ -93289,7 +93387,7 @@ "docs": { "type": "array", "items": { - "$ref": "#/components/schemas/ingest.simulate:SimulateDocumentResult" + "$ref": "#/components/schemas/ingest._types:SimulateDocumentResult" } } }, @@ -94981,6 +95079,27 @@ } } }, + "simulate.ingest#200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "docs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ingest._types:SimulateDocumentResult" + } + } + }, + "required": [ + "docs" + ] + } + } + } + }, "slm.get_lifecycle#200": { "description": "", "content": { @@ -104317,6 +104436,27 @@ }, "style": "form" }, + "simulate.ingest#index": { + "in": "path", + "name": "index", + "description": "The index to simulate ingesting into.\nThis value can be overridden by specifying an index on each document.\nIf you specify this parameter in the request path, it is used for any documents that do not explicitly specify an index argument.", + "required": true, + "deprecated": false, + "schema": { + "$ref": "#/components/schemas/_types:IndexName" + }, + "style": "simple" + }, + "simulate.ingest#pipeline": { + "in": "query", + "name": "pipeline", + "description": "The pipeline to use as the default pipeline.\nThis value can be used to override the default pipeline of the index.", + "deprecated": false, + "schema": { + "$ref": "#/components/schemas/_types:PipelineName" + }, + "style": "form" + }, "slm.get_lifecycle#policy_id": { "in": "path", "name": "policy_id", @@ -106198,7 +106338,7 @@ "description": "Sample documents to test in the pipeline.", "type": "array", "items": { - "$ref": "#/components/schemas/ingest.simulate:Document" + "$ref": "#/components/schemas/ingest._types:Document" } }, "pipeline": { @@ -107564,6 +107704,52 @@ }, "required": true }, + "simulate.ingest": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "docs": { + "description": "Sample documents to test in the pipeline.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ingest._types:Document" + } + }, + "component_template_substitutions": { + "description": "A map of component template names to substitute component template definition objects.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/cluster._types:ComponentTemplateNode" + } + }, + "index_template_subtitutions": { + "description": "A map of index template names to substitute index template definition objects.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/indices._types:IndexTemplate" + } + }, + "mapping_addition": { + "$ref": "#/components/schemas/_types.mapping:TypeMapping" + }, + "pipeline_substitutions": { + "description": "Pipelines to test.\nIf you don’t specify the `pipeline` request path parameter, this parameter is required.\nIf you specify both this and the request path parameter, the API only uses the request path parameter.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ingest._types:Pipeline" + } + } + }, + "required": [ + "docs" + ] + } + } + }, + "required": true + }, "snapshot.create": { "content": { "application/json": { diff --git a/output/openapi/elasticsearch-serverless-openapi.json b/output/openapi/elasticsearch-serverless-openapi.json index 376f1ded47..c8937a2e7d 100644 --- a/output/openapi/elasticsearch-serverless-openapi.json +++ b/output/openapi/elasticsearch-serverless-openapi.json @@ -48026,7 +48026,7 @@ "version" ] }, - "ingest.simulate:Document": { + "ingest._types:Document": { "type": "object", "properties": { "_id": { @@ -48044,11 +48044,11 @@ "_source" ] }, - "ingest.simulate:SimulateDocumentResult": { + "ingest._types:SimulateDocumentResult": { "type": "object", "properties": { "doc": { - "$ref": "#/components/schemas/ingest.simulate:DocumentSimulation" + "$ref": "#/components/schemas/ingest._types:DocumentSimulation" }, "error": { "$ref": "#/components/schemas/_types:ErrorCause" @@ -48056,12 +48056,12 @@ "processor_results": { "type": "array", "items": { - "$ref": "#/components/schemas/ingest.simulate:PipelineSimulation" + "$ref": "#/components/schemas/ingest._types:PipelineSimulation" } } } }, - "ingest.simulate:DocumentSimulation": { + "ingest._types:DocumentSimulation": { "type": "object", "properties": { "_id": { @@ -48071,7 +48071,7 @@ "$ref": "#/components/schemas/_types:IndexName" }, "_ingest": { - "$ref": "#/components/schemas/ingest.simulate:Ingest" + "$ref": "#/components/schemas/ingest._types:Ingest" }, "_routing": { "description": "Value used to send the document to a specific primary shard.", @@ -48098,11 +48098,11 @@ "_source" ] }, - "ingest.simulate:Ingest": { + "ingest._types:Ingest": { "type": "object", "properties": { "_redact": { - "$ref": "#/components/schemas/ingest.simulate:Redact" + "$ref": "#/components/schemas/ingest._types:Redact" }, "timestamp": { "$ref": "#/components/schemas/_types:DateTime" @@ -48115,7 +48115,7 @@ "timestamp" ] }, - "ingest.simulate:Redact": { + "ingest._types:Redact": { "type": "object", "properties": { "_is_redacted": { @@ -48138,11 +48138,11 @@ } ] }, - "ingest.simulate:PipelineSimulation": { + "ingest._types:PipelineSimulation": { "type": "object", "properties": { "doc": { - "$ref": "#/components/schemas/ingest.simulate:DocumentSimulation" + "$ref": "#/components/schemas/ingest._types:DocumentSimulation" }, "tag": { "type": "string" @@ -56356,7 +56356,7 @@ "docs": { "type": "array", "items": { - "$ref": "#/components/schemas/ingest.simulate:SimulateDocumentResult" + "$ref": "#/components/schemas/ingest._types:SimulateDocumentResult" } } }, @@ -63413,7 +63413,7 @@ "description": "Sample documents to test in the pipeline.", "type": "array", "items": { - "$ref": "#/components/schemas/ingest.simulate:Document" + "$ref": "#/components/schemas/ingest._types:Document" } }, "pipeline": { diff --git a/output/schema/schema.json b/output/schema/schema.json index 8d1c3f603b..6d19486213 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -18396,19 +18396,33 @@ { "availability": { "stack": { + "since": "8.12.0", "stability": "experimental", "visibility": "public" } }, - "description": "Simulates running ingest with example documents.", - "docUrl": "https://www.elastic.co/guide/en/elasticsearch/reference/master/simulate-ingest-api.html", + "description": "Simulate data ingestion.\nRun ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.\n\nThis API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.\n\nThe API runs the default and final pipeline for that index against a set of documents provided in the body of the request.\nIf a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would.\nNo data is indexed into Elasticsearch.\nInstead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation.\nThe transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.\n\nThis API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline.\nThe simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.\n\nBy default, the pipeline definitions that are currently in the system are used.\nHowever, you can supply substitute pipeline definitions in the body of the request.\nThese will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.", + "docId": "simulate-ingest-api", + "docTag": "ingest", + "docUrl": "https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/simulate-ingest-api.html", "name": "simulate.ingest", - "request": null, + "privileges": { + "index": [ + "index" + ] + }, + "request": { + "name": "Request", + "namespace": "simulate.ingest" + }, "requestBodyRequired": true, "requestMediaType": [ "application/json" ], - "response": null, + "response": { + "name": "Response", + "namespace": "simulate.ingest" + }, "responseMediaType": [ "application/json" ], @@ -144506,6 +144520,188 @@ ], "specLocation": "ingest/_types/Processors.ts#L804-L823" }, + { + "kind": "interface", + "name": { + "name": "Document", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "Unique identifier for the document.\nThis ID must be unique within the `_index`.", + "name": "_id", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "Id", + "namespace": "_types" + } + } + }, + { + "description": "Name of the index containing the document.", + "name": "_index", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "IndexName", + "namespace": "_types" + } + } + }, + { + "description": "JSON body for the document.", + "name": "_source", + "required": true, + "type": { + "kind": "user_defined_value" + } + } + ], + "specLocation": "ingest/_types/Simulation.ts#L62-L76" + }, + { + "kind": "interface", + "attachedBehaviors": [ + "AdditionalProperties" + ], + "behaviors": [ + { + "generics": [ + { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + ], + "meta": { + "description": "Additional metadata", + "fieldname": "metadata" + }, + "type": { + "name": "AdditionalProperties", + "namespace": "_spec_utils" + } + } + ], + "description": "The simulated document, with optional metadata.", + "name": { + "name": "DocumentSimulation", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "Unique identifier for the document. This ID must be unique within the `_index`.", + "name": "_id", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Id", + "namespace": "_types" + } + } + }, + { + "description": "Name of the index containing the document.", + "name": "_index", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "IndexName", + "namespace": "_types" + } + } + }, + { + "name": "_ingest", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Ingest", + "namespace": "ingest._types" + } + } + }, + { + "description": "Value used to send the document to a specific primary shard.", + "name": "_routing", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "description": "JSON body for the document.", + "name": "_source", + "required": true, + "type": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "singleKey": false, + "value": { + "kind": "user_defined_value" + } + } + }, + { + "description": "", + "name": "_version", + "required": false, + "type": { + "kind": "instance_of", + "generics": [ + { + "kind": "instance_of", + "type": { + "name": "VersionNumber", + "namespace": "_types" + } + } + ], + "type": { + "name": "Stringified", + "namespace": "_spec_utils" + } + } + }, + { + "name": "_version_type", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "VersionType", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Simulation.ts#L78-L108" + }, { "kind": "interface", "inherits": { @@ -145593,6 +145789,55 @@ ], "specLocation": "ingest/_types/Processors.ts#L1027-L1046" }, + { + "kind": "interface", + "name": { + "name": "Ingest", + "namespace": "ingest._types" + }, + "properties": [ + { + "availability": { + "serverless": {}, + "stack": { + "since": "8.16.0" + } + }, + "name": "_redact", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "Redact", + "namespace": "ingest._types" + } + } + }, + { + "name": "timestamp", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "DateTime", + "namespace": "_types" + } + } + }, + { + "name": "pipeline", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "Name", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Simulation.ts#L29-L37" + }, { "kind": "interface", "inherits": { @@ -146386,6 +146631,93 @@ ], "specLocation": "ingest/_types/Processors.ts#L1266-L1277" }, + { + "kind": "interface", + "name": { + "name": "PipelineSimulation", + "namespace": "ingest._types" + }, + "properties": [ + { + "name": "doc", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "DocumentSimulation", + "namespace": "ingest._types" + } + } + }, + { + "name": "tag", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "name": "processor_type", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "name": "status", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "ActionStatusOptions", + "namespace": "watcher._types" + } + } + }, + { + "name": "description", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "name": "ignored_error", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "ErrorCause", + "namespace": "_types" + } + } + }, + { + "name": "error", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "ErrorCause", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Simulation.ts#L52-L60" + }, { "kind": "interface", "name": { @@ -147099,6 +147431,28 @@ "nonExhaustive": true } }, + { + "kind": "interface", + "name": { + "name": "Redact", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "indicates if document has been redacted", + "name": "_is_redacted", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + } + ], + "specLocation": "ingest/_types/Simulation.ts#L39-L44" + }, { "kind": "interface", "inherits": { @@ -147695,150 +148049,49 @@ }, { "kind": "interface", - "inherits": { - "type": { - "name": "ProcessorBase", - "namespace": "ingest._types" - } - }, "name": { - "name": "SortProcessor", + "name": "SimulateDocumentResult", "namespace": "ingest._types" }, "properties": [ { - "description": "The field to be sorted.", - "name": "field", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "Field", - "namespace": "_types" - } - } - }, - { - "description": "The sort order to use.\nAccepts `\"asc\"` or `\"desc\"`.", - "name": "order", + "name": "doc", "required": false, - "serverDefault": "asc", "type": { "kind": "instance_of", "type": { - "name": "SortOrder", - "namespace": "_types" + "name": "DocumentSimulation", + "namespace": "ingest._types" } } }, { - "description": "The field to assign the sorted value to.\nBy default, the field is updated in-place.", - "name": "target_field", + "name": "error", "required": false, "type": { "kind": "instance_of", "type": { - "name": "Field", - "namespace": "_types" - } - } - } - ], - "specLocation": "ingest/_types/Processors.ts#L1478-L1494" - }, - { - "kind": "interface", - "inherits": { - "type": { - "name": "ProcessorBase", - "namespace": "ingest._types" - } - }, - "name": { - "name": "SplitProcessor", - "namespace": "ingest._types" - }, - "properties": [ - { - "description": "The field to split.", - "name": "field", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "Field", + "name": "ErrorCause", "namespace": "_types" } } }, { - "description": "If `true` and `field` does not exist, the processor quietly exits without modifying the document.", - "name": "ignore_missing", - "required": false, - "serverDefault": false, - "type": { - "kind": "instance_of", - "type": { - "name": "boolean", - "namespace": "_builtins" - } - } - }, - { - "description": "Preserves empty trailing fields, if any.", - "name": "preserve_trailing", - "required": false, - "serverDefault": false, - "type": { - "kind": "instance_of", - "type": { - "name": "boolean", - "namespace": "_builtins" - } - } - }, - { - "description": "A regex which matches the separator, for example, `,` or `\\s+`.", - "name": "separator", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - } - }, - { - "description": "The field to assign the split value to.\nBy default, the field is updated in-place.", - "name": "target_field", + "name": "processor_results", "required": false, - "serverDefault": "field", "type": { - "kind": "instance_of", - "type": { - "name": "Field", - "namespace": "_types" + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "PipelineSimulation", + "namespace": "ingest._types" + } } } } ], - "specLocation": "ingest/_types/Processors.ts#L1496-L1521" - }, - { - "kind": "interface", - "inherits": { - "type": { - "name": "ProcessorBase", - "namespace": "ingest._types" - } - }, - "name": { - "name": "TerminateProcessor", - "namespace": "ingest._types" - }, - "properties": [], - "specLocation": "ingest/_types/Processors.ts#L1523-L1523" + "specLocation": "ingest/_types/Simulation.ts#L46-L50" }, { "kind": "interface", @@ -147849,12 +148102,12 @@ } }, "name": { - "name": "TrimProcessor", + "name": "SortProcessor", "namespace": "ingest._types" }, "properties": [ { - "description": "The string-valued field to trim whitespace from.", + "description": "The field to be sorted.", "name": "field", "required": true, "type": { @@ -147866,77 +148119,22 @@ } }, { - "description": "If `true` and `field` does not exist, the processor quietly exits without modifying the document.", - "name": "ignore_missing", - "required": false, - "serverDefault": false, - "type": { - "kind": "instance_of", - "type": { - "name": "boolean", - "namespace": "_builtins" - } - } - }, - { - "description": "The field to assign the trimmed value to.\nBy default, the field is updated in-place.", - "name": "target_field", + "description": "The sort order to use.\nAccepts `\"asc\"` or `\"desc\"`.", + "name": "order", "required": false, - "serverDefault": "field", - "type": { - "kind": "instance_of", - "type": { - "name": "Field", - "namespace": "_types" - } - } - } - ], - "specLocation": "ingest/_types/Processors.ts#L1525-L1541" - }, - { - "kind": "interface", - "inherits": { - "type": { - "name": "ProcessorBase", - "namespace": "ingest._types" - } - }, - "name": { - "name": "UppercaseProcessor", - "namespace": "ingest._types" - }, - "properties": [ - { - "description": "The field to make uppercase.", - "name": "field", - "required": true, + "serverDefault": "asc", "type": { "kind": "instance_of", "type": { - "name": "Field", + "name": "SortOrder", "namespace": "_types" } } }, { - "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", - "name": "ignore_missing", - "required": false, - "serverDefault": false, - "type": { - "kind": "instance_of", - "type": { - "name": "boolean", - "namespace": "_builtins" - } - } - }, - { - "description": "The field to assign the converted value to.\nBy default, the field is updated in-place.", + "description": "The field to assign the sorted value to.\nBy default, the field is updated in-place.", "name": "target_field", "required": false, - "serverDefault": "field", "type": { "kind": "instance_of", "type": { @@ -147946,7 +148144,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1543-L1559" + "specLocation": "ingest/_types/Processors.ts#L1478-L1494" }, { "kind": "interface", @@ -147957,12 +148155,12 @@ } }, "name": { - "name": "UriPartsProcessor", + "name": "SplitProcessor", "namespace": "ingest._types" }, "properties": [ { - "description": "Field containing the URI string.", + "description": "The field to split.", "name": "field", "required": true, "type": { @@ -147987,10 +148185,10 @@ } }, { - "description": "If `true`, the processor copies the unparsed URI to `.original`.", - "name": "keep_original", + "description": "Preserves empty trailing fields, if any.", + "name": "preserve_trailing", "required": false, - "serverDefault": true, + "serverDefault": false, "type": { "kind": "instance_of", "type": { @@ -148000,23 +148198,22 @@ } }, { - "description": "If `true`, the processor removes the `field` after parsing the URI string.\nIf parsing fails, the processor does not remove the `field`.", - "name": "remove_if_successful", - "required": false, - "serverDefault": false, + "description": "A regex which matches the separator, for example, `,` or `\\s+`.", + "name": "separator", + "required": true, "type": { "kind": "instance_of", "type": { - "name": "boolean", + "name": "string", "namespace": "_builtins" } } }, { - "description": "Output field for the URI object.", + "description": "The field to assign the split value to.\nBy default, the field is updated in-place.", "name": "target_field", "required": false, - "serverDefault": "url", + "serverDefault": "field", "type": { "kind": "instance_of", "type": { @@ -148026,7 +148223,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1579-L1605" + "specLocation": "ingest/_types/Processors.ts#L1496-L1521" }, { "kind": "interface", @@ -148037,50 +148234,11 @@ } }, "name": { - "name": "UrlDecodeProcessor", + "name": "TerminateProcessor", "namespace": "ingest._types" }, - "properties": [ - { - "description": "The field to decode.", - "name": "field", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "Field", - "namespace": "_types" - } - } - }, - { - "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", - "name": "ignore_missing", - "required": false, - "serverDefault": false, - "type": { - "kind": "instance_of", - "type": { - "name": "boolean", - "namespace": "_builtins" - } - } - }, - { - "description": "The field to assign the converted value to.\nBy default, the field is updated in-place.", - "name": "target_field", - "required": false, - "serverDefault": "field", - "type": { - "kind": "instance_of", - "type": { - "name": "Field", - "namespace": "_types" - } - } - } - ], - "specLocation": "ingest/_types/Processors.ts#L1561-L1577" + "properties": [], + "specLocation": "ingest/_types/Processors.ts#L1523-L1523" }, { "kind": "interface", @@ -148091,12 +148249,254 @@ } }, "name": { - "name": "UserAgentProcessor", + "name": "TrimProcessor", "namespace": "ingest._types" }, "properties": [ { - "description": "The field containing the user agent string.", + "description": "The string-valued field to trim whitespace from.", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "If `true` and `field` does not exist, the processor quietly exits without modifying the document.", + "name": "ignore_missing", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "The field to assign the trimmed value to.\nBy default, the field is updated in-place.", + "name": "target_field", + "required": false, + "serverDefault": "field", + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Processors.ts#L1525-L1541" + }, + { + "kind": "interface", + "inherits": { + "type": { + "name": "ProcessorBase", + "namespace": "ingest._types" + } + }, + "name": { + "name": "UppercaseProcessor", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "The field to make uppercase.", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", + "name": "ignore_missing", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "The field to assign the converted value to.\nBy default, the field is updated in-place.", + "name": "target_field", + "required": false, + "serverDefault": "field", + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Processors.ts#L1543-L1559" + }, + { + "kind": "interface", + "inherits": { + "type": { + "name": "ProcessorBase", + "namespace": "ingest._types" + } + }, + "name": { + "name": "UriPartsProcessor", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "Field containing the URI string.", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "If `true` and `field` does not exist, the processor quietly exits without modifying the document.", + "name": "ignore_missing", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "If `true`, the processor copies the unparsed URI to `.original`.", + "name": "keep_original", + "required": false, + "serverDefault": true, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "If `true`, the processor removes the `field` after parsing the URI string.\nIf parsing fails, the processor does not remove the `field`.", + "name": "remove_if_successful", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "Output field for the URI object.", + "name": "target_field", + "required": false, + "serverDefault": "url", + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Processors.ts#L1579-L1605" + }, + { + "kind": "interface", + "inherits": { + "type": { + "name": "ProcessorBase", + "namespace": "ingest._types" + } + }, + "name": { + "name": "UrlDecodeProcessor", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "The field to decode.", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", + "name": "ignore_missing", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "The field to assign the converted value to.\nBy default, the field is updated in-place.", + "name": "target_field", + "required": false, + "serverDefault": "field", + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + } + ], + "specLocation": "ingest/_types/Processors.ts#L1561-L1577" + }, + { + "kind": "interface", + "inherits": { + "type": { + "name": "ProcessorBase", + "namespace": "ingest._types" + } + }, + "name": { + "name": "UserAgentProcessor", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "The field containing the user agent string.", "name": "field", "required": true, "type": { @@ -149517,346 +149917,6 @@ }, "specLocation": "ingest/put_pipeline/PutPipelineResponse.ts#L22-L24" }, - { - "kind": "interface", - "name": { - "name": "Document", - "namespace": "ingest.simulate" - }, - "properties": [ - { - "description": "Unique identifier for the document.\nThis ID must be unique within the `_index`.", - "name": "_id", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "Id", - "namespace": "_types" - } - } - }, - { - "description": "Name of the index containing the document.", - "name": "_index", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "IndexName", - "namespace": "_types" - } - } - }, - { - "description": "JSON body for the document.", - "name": "_source", - "required": true, - "type": { - "kind": "user_defined_value" - } - } - ], - "specLocation": "ingest/simulate/types.ts#L62-L76" - }, - { - "kind": "interface", - "attachedBehaviors": [ - "AdditionalProperties" - ], - "behaviors": [ - { - "generics": [ - { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - }, - { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - } - ], - "meta": { - "description": "Additional metadata", - "fieldname": "metadata" - }, - "type": { - "name": "AdditionalProperties", - "namespace": "_spec_utils" - } - } - ], - "description": "The simulated document, with optional metadata.", - "name": { - "name": "DocumentSimulation", - "namespace": "ingest.simulate" - }, - "properties": [ - { - "description": "Unique identifier for the document. This ID must be unique within the `_index`.", - "name": "_id", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "Id", - "namespace": "_types" - } - } - }, - { - "description": "Name of the index containing the document.", - "name": "_index", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "IndexName", - "namespace": "_types" - } - } - }, - { - "name": "_ingest", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "Ingest", - "namespace": "ingest.simulate" - } - } - }, - { - "description": "Value used to send the document to a specific primary shard.", - "name": "_routing", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - } - }, - { - "description": "JSON body for the document.", - "name": "_source", - "required": true, - "type": { - "kind": "dictionary_of", - "key": { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - }, - "singleKey": false, - "value": { - "kind": "user_defined_value" - } - } - }, - { - "description": "", - "name": "_version", - "required": false, - "type": { - "kind": "instance_of", - "generics": [ - { - "kind": "instance_of", - "type": { - "name": "VersionNumber", - "namespace": "_types" - } - } - ], - "type": { - "name": "Stringified", - "namespace": "_spec_utils" - } - } - }, - { - "name": "_version_type", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "VersionType", - "namespace": "_types" - } - } - } - ], - "specLocation": "ingest/simulate/types.ts#L78-L108" - }, - { - "kind": "interface", - "name": { - "name": "Ingest", - "namespace": "ingest.simulate" - }, - "properties": [ - { - "availability": { - "serverless": {}, - "stack": { - "since": "8.16.0" - } - }, - "name": "_redact", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "Redact", - "namespace": "ingest.simulate" - } - } - }, - { - "name": "timestamp", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "DateTime", - "namespace": "_types" - } - } - }, - { - "name": "pipeline", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "Name", - "namespace": "_types" - } - } - } - ], - "specLocation": "ingest/simulate/types.ts#L29-L37" - }, - { - "kind": "interface", - "name": { - "name": "PipelineSimulation", - "namespace": "ingest.simulate" - }, - "properties": [ - { - "name": "doc", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "DocumentSimulation", - "namespace": "ingest.simulate" - } - } - }, - { - "name": "tag", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - } - }, - { - "name": "processor_type", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - } - }, - { - "name": "status", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "ActionStatusOptions", - "namespace": "watcher._types" - } - } - }, - { - "name": "description", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "string", - "namespace": "_builtins" - } - } - }, - { - "name": "ignored_error", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "ErrorCause", - "namespace": "_types" - } - } - }, - { - "name": "error", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "ErrorCause", - "namespace": "_types" - } - } - } - ], - "specLocation": "ingest/simulate/types.ts#L52-L60" - }, - { - "kind": "interface", - "name": { - "name": "Redact", - "namespace": "ingest.simulate" - }, - "properties": [ - { - "description": "indicates if document has been redacted", - "name": "_is_redacted", - "required": true, - "type": { - "kind": "instance_of", - "type": { - "name": "boolean", - "namespace": "_builtins" - } - } - } - ], - "specLocation": "ingest/simulate/types.ts#L39-L44" - }, { "kind": "request", "attachedBehaviors": [ @@ -149875,7 +149935,7 @@ "kind": "instance_of", "type": { "name": "Document", - "namespace": "ingest.simulate" + "namespace": "ingest._types" } } } @@ -149949,7 +150009,7 @@ "kind": "instance_of", "type": { "name": "SimulateDocumentResult", - "namespace": "ingest.simulate" + "namespace": "ingest._types" } } } @@ -149962,52 +150022,6 @@ }, "specLocation": "ingest/simulate/SimulatePipelineResponse.ts#L22-L24" }, - { - "kind": "interface", - "name": { - "name": "SimulateDocumentResult", - "namespace": "ingest.simulate" - }, - "properties": [ - { - "name": "doc", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "DocumentSimulation", - "namespace": "ingest.simulate" - } - } - }, - { - "name": "error", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "ErrorCause", - "namespace": "_types" - } - } - }, - { - "name": "processor_results", - "required": false, - "type": { - "kind": "array_of", - "value": { - "kind": "instance_of", - "type": { - "name": "PipelineSimulation", - "namespace": "ingest.simulate" - } - } - } - } - ], - "specLocation": "ingest/simulate/types.ts#L46-L50" - }, { "kind": "interface", "name": { @@ -203649,6 +203663,179 @@ }, "specLocation": "shutdown/put_node/ShutdownPutNodeResponse.ts#L22-L24" }, + { + "kind": "request", + "attachedBehaviors": [ + "CommonQueryParameters" + ], + "body": { + "kind": "properties", + "properties": [ + { + "description": "Sample documents to test in the pipeline.", + "name": "docs", + "required": true, + "type": { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "Document", + "namespace": "ingest._types" + } + } + } + }, + { + "description": "A map of component template names to substitute component template definition objects.", + "name": "component_template_substitutions", + "required": false, + "type": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "ComponentTemplateNode", + "namespace": "cluster._types" + } + } + } + }, + { + "description": "A map of index template names to substitute index template definition objects.", + "name": "index_template_subtitutions", + "required": false, + "type": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "IndexTemplate", + "namespace": "indices._types" + } + } + } + }, + { + "name": "mapping_addition", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "TypeMapping", + "namespace": "_types.mapping" + } + } + }, + { + "description": "Pipelines to test.\nIf you don’t specify the `pipeline` request path parameter, this parameter is required.\nIf you specify both this and the request path parameter, the API only uses the request path parameter.", + "name": "pipeline_substitutions", + "required": false, + "type": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "Pipeline", + "namespace": "ingest._types" + } + } + } + } + ] + }, + "description": "Simulate data ingestion.\nRun ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index.\n\nThis API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch.\n\nThe API runs the default and final pipeline for that index against a set of documents provided in the body of the request.\nIf a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would.\nNo data is indexed into Elasticsearch.\nInstead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation.\nThe transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result.\n\nThis API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline.\nThe simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index.\n\nBy default, the pipeline definitions that are currently in the system are used.\nHowever, you can supply substitute pipeline definitions in the body of the request.\nThese will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request.", + "inherits": { + "type": { + "name": "RequestBase", + "namespace": "_types" + } + }, + "name": { + "name": "Request", + "namespace": "simulate.ingest" + }, + "path": [ + { + "description": "The index to simulate ingesting into.\nThis value can be overridden by specifying an index on each document.\nIf you specify this parameter in the request path, it is used for any documents that do not explicitly specify an index argument.", + "name": "index", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "IndexName", + "namespace": "_types" + } + } + } + ], + "query": [ + { + "description": "The pipeline to use as the default pipeline.\nThis value can be used to override the default pipeline of the index.", + "name": "pipeline", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "PipelineName", + "namespace": "_types" + } + } + } + ], + "specLocation": "simulate/ingest/SimulateIngestRequest.ts#L29-L90" + }, + { + "kind": "response", + "body": { + "kind": "properties", + "properties": [ + { + "name": "docs", + "required": true, + "type": { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "SimulateDocumentResult", + "namespace": "ingest._types" + } + } + } + } + ] + }, + "name": { + "name": "Response", + "namespace": "simulate.ingest" + }, + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L22-L24" + }, { "kind": "interface", "name": { diff --git a/output/schema/validation-errors.json b/output/schema/validation-errors.json index fb2c347217..ff0d9d3d00 100644 --- a/output/schema/validation-errors.json +++ b/output/schema/validation-errors.json @@ -907,12 +907,6 @@ ], "response": [] }, - "simulate.ingest": { - "request": [ - "Missing request & response" - ], - "response": [] - }, "snapshot.clone": { "request": [ "Request: query parameter 'timeout' does not exist in the json spec" diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 9c01c72da8..f0384031e0 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -12997,6 +12997,24 @@ export interface IngestDissectProcessor extends IngestProcessorBase { pattern: string } +export interface IngestDocument { + _id?: Id + _index?: IndexName + _source: any +} + +export interface IngestDocumentSimulationKeys { + _id: Id + _index: IndexName + _ingest: IngestIngest + _routing?: string + _source: Record + _version?: SpecUtilsStringified + _version_type?: VersionType +} +export type IngestDocumentSimulation = IngestDocumentSimulationKeys + & { [property: string]: string | Id | IndexName | IngestIngest | Record | SpecUtilsStringified | VersionType } + export interface IngestDotExpanderProcessor extends IngestProcessorBase { field: Field override?: boolean @@ -13110,6 +13128,12 @@ export interface IngestInferenceProcessor extends IngestProcessorBase { inference_config?: IngestInferenceConfig } +export interface IngestIngest { + _redact?: IngestRedact + timestamp: DateTime + pipeline?: Name +} + export interface IngestIpLocationProcessor extends IngestProcessorBase { database_file?: string field: Field @@ -13197,6 +13221,16 @@ export interface IngestPipelineProcessor extends IngestProcessorBase { ignore_missing_pipeline?: boolean } +export interface IngestPipelineSimulation { + doc?: IngestDocumentSimulation + tag?: string + processor_type?: string + status?: WatcherActionStatusOptions + description?: string + ignored_error?: ErrorCause + error?: ErrorCause +} + export interface IngestProcessorBase { description?: string if?: string @@ -13253,6 +13287,10 @@ export interface IngestProcessorContainer { user_agent?: IngestUserAgentProcessor } +export interface IngestRedact { + _is_redacted: boolean +} + export interface IngestRedactProcessor extends IngestProcessorBase { field: Field patterns: GrokPattern[] @@ -13311,6 +13349,12 @@ export interface IngestSetSecurityUserProcessor extends IngestProcessorBase { export type IngestShapeType = 'geo_shape' | 'shape' +export interface IngestSimulateDocumentResult { + doc?: IngestDocumentSimulation + error?: ErrorCause + processor_results?: IngestPipelineSimulation[] +} + export interface IngestSortProcessor extends IngestProcessorBase { field: Field order?: SortOrder @@ -13505,61 +13549,17 @@ export interface IngestPutPipelineRequest extends RequestBase { export type IngestPutPipelineResponse = AcknowledgedResponseBase -export interface IngestSimulateDocument { - _id?: Id - _index?: IndexName - _source: any -} - -export interface IngestSimulateDocumentSimulationKeys { - _id: Id - _index: IndexName - _ingest: IngestSimulateIngest - _routing?: string - _source: Record - _version?: SpecUtilsStringified - _version_type?: VersionType -} -export type IngestSimulateDocumentSimulation = IngestSimulateDocumentSimulationKeys - & { [property: string]: string | Id | IndexName | IngestSimulateIngest | Record | SpecUtilsStringified | VersionType } - -export interface IngestSimulateIngest { - _redact?: IngestSimulateRedact - timestamp: DateTime - pipeline?: Name -} - -export interface IngestSimulatePipelineSimulation { - doc?: IngestSimulateDocumentSimulation - tag?: string - processor_type?: string - status?: WatcherActionStatusOptions - description?: string - ignored_error?: ErrorCause - error?: ErrorCause -} - -export interface IngestSimulateRedact { - _is_redacted: boolean -} - export interface IngestSimulateRequest extends RequestBase { id?: Id verbose?: boolean body?: { - docs: IngestSimulateDocument[] + docs: IngestDocument[] pipeline?: IngestPipeline } } export interface IngestSimulateResponse { - docs: IngestSimulateSimulateDocumentResult[] -} - -export interface IngestSimulateSimulateDocumentResult { - doc?: IngestSimulateDocumentSimulation - error?: ErrorCause - processor_results?: IngestSimulatePipelineSimulation[] + docs: IngestSimulateDocumentResult[] } export interface LicenseLicense { @@ -19170,6 +19170,22 @@ export interface ShutdownPutNodeRequest extends RequestBase { export type ShutdownPutNodeResponse = AcknowledgedResponseBase +export interface SimulateIngestRequest extends RequestBase { + index?: IndexName + pipeline?: PipelineName + body?: { + docs: IngestDocument[] + component_template_substitutions?: Record + index_template_subtitutions?: Record + mapping_addition?: MappingTypeMapping + pipeline_substitutions?: Record + } +} + +export interface SimulateIngestResponse { + docs: IngestSimulateDocumentResult[] +} + export interface SlmConfiguration { ignore_unavailable?: boolean indices?: Indices diff --git a/specification/_doc_ids/table.csv b/specification/_doc_ids/table.csv index 07abc7ecce..26d295a8a1 100644 --- a/specification/_doc_ids/table.csv +++ b/specification/_doc_ids/table.csv @@ -647,6 +647,7 @@ security-privileges,https://www.elastic.co/guide/en/elasticsearch/reference/{bra service-accounts,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/service-accounts.html set-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/set-processor.html shape,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/shape.html +simulate-ingest-api,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/simulate-ingest-api.html simulate-pipeline-api,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/simulate-pipeline-api.html slm-api-delete-policy,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/slm-api-delete-policy.html slm-api-execute-lifecycle,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/slm-api-execute-lifecycle.html diff --git a/specification/ingest/simulate/types.ts b/specification/ingest/_types/Simulation.ts similarity index 100% rename from specification/ingest/simulate/types.ts rename to specification/ingest/_types/Simulation.ts diff --git a/specification/ingest/simulate/SimulatePipelineRequest.ts b/specification/ingest/simulate/SimulatePipelineRequest.ts index ba29c492d5..a1446b4ebe 100644 --- a/specification/ingest/simulate/SimulatePipelineRequest.ts +++ b/specification/ingest/simulate/SimulatePipelineRequest.ts @@ -20,7 +20,7 @@ import { Pipeline } from '@ingest/_types/Pipeline' import { RequestBase } from '@_types/Base' import { Id } from '@_types/common' -import { Document } from './types' +import { Document } from '../_types/Simulation' /** * Simulate a pipeline. diff --git a/specification/ingest/simulate/SimulatePipelineResponse.ts b/specification/ingest/simulate/SimulatePipelineResponse.ts index 6402805200..effc939bae 100644 --- a/specification/ingest/simulate/SimulatePipelineResponse.ts +++ b/specification/ingest/simulate/SimulatePipelineResponse.ts @@ -17,7 +17,7 @@ * under the License. */ -import { SimulateDocumentResult } from './types' +import { SimulateDocumentResult } from '../_types/Simulation' export class Response { body: { docs: SimulateDocumentResult[] } diff --git a/specification/ingest/simulate/SimulatePipelineRequestExample1.yaml b/specification/ingest/simulate/examples/request/SimulatePipelineRequestExample1.yaml similarity index 100% rename from specification/ingest/simulate/SimulatePipelineRequestExample1.yaml rename to specification/ingest/simulate/examples/request/SimulatePipelineRequestExample1.yaml diff --git a/specification/ingest/simulate/SimulatePipelineResponseExample1.yaml b/specification/ingest/simulate/examples/response/SimulatePipelineResponseExample1.yaml similarity index 100% rename from specification/ingest/simulate/SimulatePipelineResponseExample1.yaml rename to specification/ingest/simulate/examples/response/SimulatePipelineResponseExample1.yaml diff --git a/specification/simulate/ingest/SimulateIngestRequest.ts b/specification/simulate/ingest/SimulateIngestRequest.ts new file mode 100644 index 0000000000..f545c359d6 --- /dev/null +++ b/specification/simulate/ingest/SimulateIngestRequest.ts @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ComponentTemplateNode } from '@cluster/_types/ComponentTemplate' +import { IndexTemplate } from '@indices/_types/IndexTemplate' +import { Pipeline } from '@ingest/_types/Pipeline' +import { Document } from '@ingest/_types/Simulation' +import { Dictionary } from '@spec_utils/Dictionary' +import { RequestBase } from '@_types/Base' +import { IndexName, PipelineName } from '@_types/common' +import { TypeMapping } from '@_types/mapping/TypeMapping' + +/** + * Simulate data ingestion. + * Run ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions, to simulate ingesting data into an index. + * + * This API is meant to be used for troubleshooting or pipeline development, as it does not actually index any data into Elasticsearch. + * + * The API runs the default and final pipeline for that index against a set of documents provided in the body of the request. + * If a pipeline contains a reroute processor, it follows that reroute processor to the new index, running that index's pipelines as well the same way that a non-simulated ingest would. + * No data is indexed into Elasticsearch. + * Instead, the transformed document is returned, along with the list of pipelines that have been run and the name of the index where the document would have been indexed if this were not a simulation. + * The transformed document is validated against the mappings that would apply to this index, and any validation error is reported in the result. + * + * This API differs from the simulate pipeline API in that you specify a single pipeline for that API, and it runs only that one pipeline. + * The simulate pipeline API is more useful for developing a single pipeline, while the simulate ingest API is more useful for troubleshooting the interaction of the various pipelines that get applied when ingesting into an index. + * + * By default, the pipeline definitions that are currently in the system are used. + * However, you can supply substitute pipeline definitions in the body of the request. + * These will be used in place of the pipeline definitions that are already in the system. This can be used to replace existing pipeline definitions or to create new ones. The pipeline substitutions are used only within this request. + * @rest_spec_name simulate.ingest + * @availability stack since=8.12.0 stability=experimental visibility=public + * @index_privileges index + * @doc_tag ingest + * @doc_id simulate-ingest-api + */ +export interface Request extends RequestBase { + path_parts: { + /** + * The index to simulate ingesting into. + * This value can be overridden by specifying an index on each document. + * If you specify this parameter in the request path, it is used for any documents that do not explicitly specify an index argument. + */ + index?: IndexName + } + query_parameters: { + /** + * The pipeline to use as the default pipeline. + * This value can be used to override the default pipeline of the index. + */ + pipeline?: PipelineName + } + body: { + /** + * Sample documents to test in the pipeline. + */ + docs: Document[] + /** + * A map of component template names to substitute component template definition objects. + */ + component_template_substitutions?: Dictionary + /** + * A map of index template names to substitute index template definition objects. + */ + index_template_subtitutions?: Dictionary + mapping_addition?: TypeMapping + /** + * Pipelines to test. + * If you don’t specify the `pipeline` request path parameter, this parameter is required. + * If you specify both this and the request path parameter, the API only uses the request path parameter. + */ + pipeline_substitutions?: Dictionary + } +} diff --git a/specification/simulate/ingest/SimulateIngestResponse.ts b/specification/simulate/ingest/SimulateIngestResponse.ts new file mode 100644 index 0000000000..e1812124fb --- /dev/null +++ b/specification/simulate/ingest/SimulateIngestResponse.ts @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SimulateDocumentResult } from '@ingest/_types/Simulation' + +export class Response { + body: { docs: SimulateDocumentResult[] } +} diff --git a/specification/simulate/ingest/examples/request/SimulateIngestRequestExample1.yaml b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample1.yaml new file mode 100644 index 0000000000..3cc3266ea2 --- /dev/null +++ b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample1.yaml @@ -0,0 +1,14 @@ +summary: Existing pipeline definitions +# method_request: POST /_ingest/_simulate +description: In this example the index `my-index` has a default pipeline called `my-pipeline` and a final pipeline called `my-final-pipeline`. Since both documents are being ingested into `my-index`, both pipelines are run using the pipeline definitions that are already in the system. +# type: request +value: + docs: + - _id: 123 + _index: my-index + _source: + foo: bar + - _id: 456 + _index: my-index + _source: + foo: rab diff --git a/specification/simulate/ingest/examples/request/SimulateIngestRequestExample2.yaml b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample2.yaml new file mode 100644 index 0000000000..2c58625056 --- /dev/null +++ b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample2.yaml @@ -0,0 +1,19 @@ +summary: Pipeline substitions +# method_request: POST /_ingest/_simulate +description: In this example the index `my-index` has a default pipeline called `my-pipeline` and a final pipeline called `my-final-pipeline`. But a substitute definition of `my-pipeline` is provided in `pipeline_substitutions`. The substitute `my-pipeline` will be used in place of the `my-pipeline` that is in the system, and then the `my-final-pipeline` that is already defined in the system will run. +# type: request +value: + docs: + - _index: my-index + _id: 123 + _source: + foo: bar + - _index: my-index + _id: 456 + _source: + foo: rab + pipeline_substitutions: + my-pipeline: + processors: + - uppercase: + field: foo diff --git a/specification/simulate/ingest/examples/request/SimulateIngestRequestExample3.yaml b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample3.yaml new file mode 100644 index 0000000000..e52b598e14 --- /dev/null +++ b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample3.yaml @@ -0,0 +1,44 @@ +summary: Component template substitutions +# method_request: POST /_ingest/_simulate +description: > + In this example, imagine that the index `my-index` has a strict mapping with only the `foo` keyword field defined. + Say that field mapping came from a component template named `my-mappings-template`. You want to test adding a new field, `bar`. + So a substitute definition of `my-mappings-template` is provided in `component_template_substitutions`. + The substitute `my-mappings-template` will be used in place of the existing mapping for `my-index` and in place of the `my-mappings-template` that is in the system. +# type: request +value: |- + { + "docs": [ + { + "_index": "my-index", + "_id": "123", + "_source": { + "foo": "foo" + } + }, + { + "_index": "my-index", + "_id": "456", + "_source": { + "bar": "rab" + } + } + ], + "component_template_substitutions": { + "my-mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } + } + } + } + } + } + } diff --git a/specification/simulate/ingest/examples/request/SimulateIngestRequestExample4.yaml b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample4.yaml new file mode 100644 index 0000000000..5e0fed2ae8 --- /dev/null +++ b/specification/simulate/ingest/examples/request/SimulateIngestRequestExample4.yaml @@ -0,0 +1,43 @@ +summary: Multiple substitutions +# method_request: POST /_ingest/_simulate +description: The pipeline, component template, and index template substitutions replace the existing pipeline details for the duration of this request. +# type: request +value: + docs: + - _id: id + _index: my-index + _source: + foo: bar + - _id: id + _index: my-index + _source: + foo: rab + pipeline_substitutions: + my-pipeline: + processors: + - set: + field: field3 + value: value3 + component_template_substitutions: + my-component-template: + template: + mappings: + dynamic: true + properties: + field3: + type: keyword + settings: + index: + default_pipeline: my-pipeline + index_template_substitutions: + my-index-template: + index_patterns: + - 'my-index-*' + composed_of: + - component_template_1 + - component_template_2 + mapping_addition: + dynamic: strict + properties: + foo: + type: keyword diff --git a/specification/simulate/ingest/examples/response/SimulateIngestResponseExample1.yaml b/specification/simulate/ingest/examples/response/SimulateIngestResponseExample1.yaml new file mode 100644 index 0000000000..7d17f46f84 --- /dev/null +++ b/specification/simulate/ingest/examples/response/SimulateIngestResponseExample1.yaml @@ -0,0 +1,28 @@ +summary: Use an existing pipeline definition +description: A successful response when the simulation uses pipeline definitions that are already in the system. +# type: response +# response_code: 200 +value: + docs: + - doc: + _id: 123 + _index: my-index + _version: -3 + _source: + field1: value1 + field2: value2 + foo: bar + executed_pipelines: + - my-pipeline + - my-final-pipeline + - doc: + _id: 456 + _index: my-index + _version: -3, + _source: + field1: value1 + field2: value2 + foo: rab + executed_pipelines: + - my-pipeline + - my-final-pipeline diff --git a/specification/simulate/ingest/examples/response/SimulateIngestResponseExample2.yaml b/specification/simulate/ingest/examples/response/SimulateIngestResponseExample2.yaml new file mode 100644 index 0000000000..d4274fbbab --- /dev/null +++ b/specification/simulate/ingest/examples/response/SimulateIngestResponseExample2.yaml @@ -0,0 +1,26 @@ +summary: Use pipeline substitutions +description: A successful response when the simulation uses pipeline substitutions. +# type: response +# response_code: 200 +value: + docs: + - doc: + _id: 123 + _index: my-index + _version: -3 + _source: + field2: value2 + foo: BAR + executed_pipelines: + - my-pipeline + - my-final-pipeline + - doc: + _id: 456 + _index: my-index + _version: -3 + _source: + field2: value2 + foo: RAB + executed_pipelines: + - my-pipeline + - my-final-pipeline diff --git a/specification/simulate/ingest/examples/response/SimulateIngestResponseExample3.yaml b/specification/simulate/ingest/examples/response/SimulateIngestResponseExample3.yaml new file mode 100644 index 0000000000..fdd1f30ec3 --- /dev/null +++ b/specification/simulate/ingest/examples/response/SimulateIngestResponseExample3.yaml @@ -0,0 +1,31 @@ +summary: Use pipeline substitutions +description: A successful response when the simulation uses pipeline substitutions. +# type: response +# response_code: 200 +value: |- + { + "docs": [ + { + "doc": { + "_id": "123", + "_index": "my-index", + "_version": -3, + "_source": { + "foo": "foo" + }, + "executed_pipelines": [] + } + }, + { + "doc": { + "_id": "456", + "_index": "my-index", + "_version": -3, + "_source": { + "bar": "rab" + }, + "executed_pipelines": [] + } + } + ] + }