Adding distributed components

assets/training/finetune_acft_hf_nlp/components/finetune/chat_completion/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: chat_completion_finetune
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/finetune/question_answering/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: question_answering_finetune
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/finetune/summarization/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: summarization_finetune
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/finetune/text_classification/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: text_classification_finetune
-version: 0.0.66
+version: 0.0.67
 type: command
 
 is_deterministic: false

assets/training/finetune_acft_hf_nlp/components/finetune/text_generation/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: text_generation_finetune
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/finetune/token_classification/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: token_classification_finetune
-version: 0.0.66
+version: 0.0.67
 type: command
 
 is_deterministic: false

assets/training/finetune_acft_hf_nlp/components/finetune/translation/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: translation_finetune
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_converter/common/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: ft_nlp_model_converter
-version: 0.0.66
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/chat_completion/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: chat_completion_model_import
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/question_answering/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: question_answering_model_import
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/summarization/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: summarization_model_import
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/text_classification/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: text_classification_model_import
-version: 0.0.66
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/text_generation/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: text_generation_model_import
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/token_classification/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: token_classification_model_import
-version: 0.0.66
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/model_import/translation/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: translation_model_import
-version: 0.0.65
+version: 0.0.67
 type: command
 
 is_deterministic: True

assets/training/finetune_acft_hf_nlp/components/pipeline_components/chat_completion/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json
 name: chat_completion_pipeline
-version: 0.0.67
+version: 0.0.68
 type: pipeline
 display_name: Chat Completion Pipeline
 description: Pipeline Component to finetune Hugging Face pretrained models for chat completion task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/chat_completion_pipeline) to learn more.
@@ -495,7 +495,7 @@ outputs:
 jobs:
   ft_nlp_common_validation:
     type: command
-    component: azureml:ft_nlp_common_validation:0.0.65
+    component: azureml:ft_nlp_common_validation:0.0.67
     compute: '${{parent.inputs.compute_model_import}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_import}}'
@@ -532,7 +532,7 @@ jobs:
       auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}'
   chat_completion_model_import:
     type: command
-    component: azureml:chat_completion_model_import:0.0.65
+    component: azureml:chat_completion_model_import:0.0.67
     compute: '${{parent.inputs.compute_model_import}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_import}}'
@@ -543,7 +543,7 @@ jobs:
       validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}'
   chat_completion_datapreprocess:
     type: command
-    component: azureml:chat_completion_datapreprocess:0.0.65
+    component: azureml:chat_completion_datapreprocess:0.0.67
     compute: '${{parent.inputs.compute_preprocess}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_preprocess}}'
@@ -560,7 +560,7 @@ jobs:
       model_selector_output: '${{parent.jobs.chat_completion_model_import.outputs.output_dir}}'
   chat_completion_finetune:
     type: command
-    component: azureml:chat_completion_finetune:0.0.65
+    component: azureml:chat_completion_finetune:0.0.67
     compute: '${{parent.inputs.compute_finetune}}'
     distribution:
       type: pytorch
@@ -618,7 +618,7 @@ jobs:
       # mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}'
   ft_nlp_model_converter:
     type: command
-    component: azureml:ft_nlp_model_converter:0.0.65
+    component: azureml:ft_nlp_model_converter:0.0.67
     compute: '${{parent.inputs.compute_finetune}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_finetune}}'
@@ -627,30 +627,30 @@ jobs:
       # converted_model: '${{parent.jobs.chat_completion_finetune.outputs.mlflow_model_folder}}'
     outputs:
       mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}'
-  # model_prediction:
-  #   type: command
-  #   component: azureml:model_prediction_with_container:0.0.2
-  #   compute: '${{parent.inputs.compute_model_evaluation}}'
-  #   resources:
-  #     instance_type: '${{parent.inputs.instance_type_model_evaluation}}'
-  #   inputs:
-  #     task: chat-completion
-  #     test_data: '${{parent.jobs.chat_completion_datapreprocess.outputs.output_dir}}'
-  #     label_column_name: messages
-  #     mlflow_model: '${{parent.jobs.ft_nlp_model_converter.outputs.mlflow_model_folder}}'
-  #     evaluation_config_params: '${{parent.inputs.evaluation_config_params}}'
-  # compute_metrics:
-  #   type: command
-  #   component: azureml:compute_metrics:0.0.31
-  #   compute: '${{parent.inputs.compute_model_evaluation}}'
-  #   resources:
-  #     instance_type: '${{parent.inputs.instance_type_model_evaluation}}'
-  #   inputs:
-  #     task: chat-completion
-  #     ground_truth: '${{parent.jobs.model_prediction.outputs.ground_truth}}'
-  #     prediction: '${{parent.jobs.model_prediction.outputs.predictions}}'
-  #     prediction_probabilities: '${{parent.jobs.model_prediction.outputs.prediction_probabilities}}'
-  #     evaluation_config: '${{parent.inputs.evaluation_config}}'
-  #     evaluation_config_params: '${{parent.inputs.evaluation_config_params}}'
-  #   outputs:
-  #     evaluation_result: '${{parent.outputs.evaluation_result}}'
+  model_prediction:
+    type: command
+    component: azureml:model_prediction_with_container:0.0.5
+    compute: '${{parent.inputs.compute_model_evaluation}}'
+    resources:
+      instance_type: '${{parent.inputs.instance_type_model_evaluation}}'
+    inputs:
+      task: chat-completion
+      test_data: '${{parent.jobs.chat_completion_datapreprocess.outputs.output_dir}}'
+      label_column_name: messages
+      mlflow_model: '${{parent.jobs.ft_nlp_model_converter.outputs.mlflow_model_folder}}'
+      evaluation_config_params: '${{parent.inputs.evaluation_config_params}}'
+  compute_metrics:
+    type: command
+    component: azureml:compute_metrics:0.0.31
+    compute: '${{parent.inputs.compute_model_evaluation}}'
+    resources:
+      instance_type: '${{parent.inputs.instance_type_model_evaluation}}'
+    inputs:
+      task: chat-completion
+      ground_truth: '${{parent.jobs.model_prediction.outputs.ground_truth}}'
+      prediction: '${{parent.jobs.model_prediction.outputs.predictions}}'
+      prediction_probabilities: '${{parent.jobs.model_prediction.outputs.prediction_probabilities}}'
+      evaluation_config: '${{parent.inputs.evaluation_config}}'
+      evaluation_config_params: '${{parent.inputs.evaluation_config_params}}'
+    outputs:
+      evaluation_result: '${{parent.outputs.evaluation_result}}'

assets/training/finetune_acft_hf_nlp/components/pipeline_components/question_answering/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json
 name: question_answering_pipeline
-version: 0.0.67
+version: 0.0.68
 type: pipeline
 display_name: Question Answering Pipeline
 description: Pipeline Component to finetune Hugging Face pretrained models for extractive question answering task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/question_answering_pipeline) to learn more.
@@ -541,7 +541,7 @@ outputs:
 jobs:
   ft_nlp_common_validation:
     type: command
-    component: azureml:ft_nlp_common_validation:0.0.64
+    component: azureml:ft_nlp_common_validation:0.0.67
     compute: '${{parent.inputs.compute_model_import}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_import}}'
@@ -578,7 +578,7 @@ jobs:
       auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}'
   question_answering_model_import:
     type: command
-    component: azureml:question_answering_model_import:0.0.65
+    component: azureml:question_answering_model_import:0.0.67
     compute: '${{parent.inputs.compute_model_import}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_import}}'
@@ -589,7 +589,7 @@ jobs:
       validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}'
   question_answering_datapreprocess:
     type: command
-    component: azureml:question_answering_datapreprocess:0.0.65
+    component: azureml:question_answering_datapreprocess:0.0.67
     compute: '${{parent.inputs.compute_preprocess}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_preprocess}}'
@@ -614,7 +614,7 @@ jobs:
       model_selector_output: '${{parent.jobs.question_answering_model_import.outputs.output_dir}}'
   question_answering_finetune:
     type: command
-    component: azureml:question_answering_finetune:0.0.65
+    component: azureml:question_answering_finetune:0.0.67
     compute: '${{parent.inputs.compute_finetune}}'
     distribution:
       type: pytorch
@@ -671,7 +671,7 @@ jobs:
       pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}'
   ft_nlp_model_converter:
     type: command
-    component: azureml:ft_nlp_model_converter:0.0.64
+    component: azureml:ft_nlp_model_converter:0.0.67
     compute: '${{parent.inputs.compute_finetune}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_finetune}}'
@@ -681,7 +681,7 @@ jobs:
       mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}'
   model_prediction:
     type: command
-    component: azureml:model_prediction:0.0.35
+    component: azureml:model_prediction_with_container:0.0.5
     compute: '${{parent.inputs.compute_model_evaluation}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_evaluation}}'
@@ -690,10 +690,7 @@ jobs:
       test_data: '${{parent.jobs.question_answering_datapreprocess.outputs.output_dir}}'
       label_column_name: '${{parent.inputs.answers_key}}'
       input_column_names: "'${{parent.inputs.question_key}}','${{parent.inputs.context_key}}'"
-      batch_size: '${{parent.inputs.per_device_train_batch_size}}'
-      device: auto
       mlflow_model: '${{parent.jobs.ft_nlp_model_converter.outputs.mlflow_model_folder}}'
-      evaluation_config: '${{parent.inputs.evaluation_config}}'
       evaluation_config_params: '${{parent.inputs.evaluation_config_params}}'
   compute_metrics:
     type: command

assets/training/finetune_acft_hf_nlp/components/pipeline_components/summarization/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json
 name: summarization_pipeline
-version: 0.0.67
+version: 0.0.68
 type: pipeline
 display_name: Summarization Pipeline
 description: Pipeline Component to finetune Hugging Face pretrained models for summarization task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/summarization_pipeline) to learn more.
@@ -512,7 +512,7 @@ outputs:
 jobs:
   ft_nlp_common_validation:
     type: command
-    component: azureml:ft_nlp_common_validation:0.0.64
+    component: azureml:ft_nlp_common_validation:0.0.67
     compute: '${{parent.inputs.compute_model_import}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_import}}'
@@ -549,7 +549,7 @@ jobs:
       auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}'
   summarization_model_import:
     type: command
-    component: azureml:summarization_model_import:0.0.65
+    component: azureml:summarization_model_import:0.0.67
     compute: '${{parent.inputs.compute_model_import}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_import}}'
@@ -560,7 +560,7 @@ jobs:
       validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}'
   summarization_datapreprocess:
     type: command
-    component: azureml:summarization_datapreprocess:0.0.65
+    component: azureml:summarization_datapreprocess:0.0.67
     compute: '${{parent.inputs.compute_preprocess}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_preprocess}}'
@@ -580,7 +580,7 @@ jobs:
       model_selector_output: '${{parent.jobs.summarization_model_import.outputs.output_dir}}'
   summarization_finetune:
     type: command
-    component: azureml:summarization_finetune:0.0.65
+    component: azureml:summarization_finetune:0.0.67
     compute: '${{parent.inputs.compute_finetune}}'
     distribution:
       type: pytorch
@@ -637,7 +637,7 @@ jobs:
       pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}'
   ft_nlp_model_converter:
     type: command
-    component: azureml:ft_nlp_model_converter:0.0.64
+    component: azureml:ft_nlp_model_converter:0.0.67
     compute: '${{parent.inputs.compute_finetune}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_finetune}}'
@@ -647,7 +647,7 @@ jobs:
       mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}'
   model_prediction:
     type: command
-    component: azureml:model_prediction:0.0.35
+    component: azureml:model_prediction_with_container:0.0.5
     compute: '${{parent.inputs.compute_model_evaluation}}'
     resources:
       instance_type: '${{parent.inputs.instance_type_model_evaluation}}'
@@ -656,10 +656,7 @@ jobs:
       test_data: '${{parent.jobs.summarization_datapreprocess.outputs.output_dir}}'
       label_column_name: '${{parent.inputs.summary_key}}'
       input_column_names: '${{parent.inputs.document_key}}'
-      batch_size: '${{parent.inputs.per_device_train_batch_size}}'
-      device: auto
       mlflow_model: '${{parent.jobs.ft_nlp_model_converter.outputs.mlflow_model_folder}}'
-      evaluation_config: '${{parent.inputs.evaluation_config}}'
       evaluation_config_params: '${{parent.inputs.evaluation_config_params}}'
   compute_metrics:
     type: command