Enable n300 model tests. Use to create new processes with python mult…

…iprocessing rather than the default

make sure to terminate process if an error is raised before we get to  it

.github/workflows/run-model-tests.yml

@@ -16,6 +16,11 @@ jobs:
         build: [
           {runs-on: n150, name: "run1", test_names: "stable_diffusion, Qwen, MobileNetV2, clip, flan_t5, mlpmixer, resnet, vilt, albert, codegen, glpn_kitti, mnist, resnet50, RMBG, unet_carvana, mgp-str-base, musicgen_small, segformer, torchvision, yolos"},
           {runs-on: n150, name: "run2", test_names: "t5, whisper, autoencoder_conv, deit, gpt2, mobilenet_ssd, roberta, timm, xglm, autoencoder_linear, detr, beit, distilbert, hand_landmark, openpose, segment_anything, unet, yolov3, bert, dpr, hardnet, opt, speecht5_tts, unet_brain, yolov5, bloom, falcon, llama, perceiver_io, squeeze_bert, gpt_neo"},
+
+          {runs-on: n300, name: "run1", test_names: "stable_diffusion, Qwen, MobileNetV2, clip, flan_t5, mlpmixer, resnet, vilt, albert, codegen"},
+          {runs-on: n300, name: "run2", test_names: "glpn_kitti, mnist, resnet50, RMBG, unet_carvana, mgp-str-base, musicgen_small, segformer, torchvision, yolos"},
+          {runs-on: n300, name: "run3", test_names: "t5, whisper, autoencoder_conv, deit, gpt2, mobilenet_ssd, roberta, timm, xglm, autoencoder_linear, detr, beit, distilbert, hand_landmark, openpose, segment_anything, unet, "},
+          {runs-on: n300, name: "run4", test_names: "yolov3, bert, dpr, hardnet, opt, speecht5_tts, unet_brain, yolov5, bloom, falcon, llama, perceiver_io, squeeze_bert, gpt_neo"},
         ]
     runs-on:
       - ${{ matrix.build.runs-on }}

.github/workflows/run-tests.yml

@@ -15,6 +15,7 @@ jobs:
       matrix:
         build: [
           {runs-on: n150, name: "run"},
+          {runs-on: n300, name: "run"},
         ]
 
     runs-on:

tests/conftest.py

@@ -4,6 +4,9 @@
 
 import pytest
 import torch
+import multiprocessing as mp
+
+mp.set_start_method("spawn")
 
 
 @pytest.fixture(autouse=True)

tt_torch/tools/utils.py

@@ -128,7 +128,7 @@ def __init__(self):
         self.stable_hlo_ops = []
         self.model_name = ""
         self.results_path = "results/models/"
-        self.single_op_timeout = 5
+        self.single_op_timeout = 30
         self.enable_intermediate_verification = False
         self.enable_consteval = False
         self.remove_embedded_constants = False