Merge branch 'main' into meng_fix

.github/ci_expected_accuracy/check_expected.py

@@ -17,14 +17,14 @@
 
 
 # load csv files
-test_data= pd.read_csv(args.csv_file, comment='#')
+test_data = pd.read_csv(args.csv_file, comment='#')
 # test_data = test_data.reset_index()  # make sure indexes pair with number of rows
 # test_data = test_data.sort_values(by=["name"], ascending=True)
 test_names = [row["name"] for index, row in test_data.iterrows()]
 
 current_path = pathlib.Path(__file__).parent.resolve()
 refer_file = str(current_path) + "/" + args.category + "_" + args.suite + "_" + args.mode + ".csv"
-refer_data= pd.read_csv(refer_file, comment='#')
+refer_data = pd.read_csv(refer_file, comment='#')
 # refer_data = refer_data.reset_index()  # make sure indexes pair with number of rows
 # refer_data = refer_data.sort_values(by=["name"], ascending=True)
 refer_names = [row["name"] for index, row in refer_data.iterrows()]
@@ -38,8 +38,8 @@
 new_pass_models = []
 lost_models = []
 timeout_models = []
-for model_name in model_names:
 # for index, row in refer_data.iterrows():
+for model_name in model_names:
     test_row = next(([i, line] for i, line in test_data.iterrows() if line["name"] == model_name), "N/A")
     refer_row = next(([i, line] for i, line in refer_data.iterrows() if line["name"] == model_name), "N/A")
     test_accuracy = test_row[1]["accuracy"] if test_row != "N/A" else "N/A"
@@ -52,7 +52,7 @@
         passed_models.append([model_name, test_accuracy])
         if refer_accuracy == "N/A":
             new_models.append([model_name, test_accuracy])
-            refer_data.loc[len(refer_data),:] = "N/A"
+            refer_data.loc[len(refer_data), :] = "N/A"
             refer_data.at[len(refer_data) - 1, "name"] = model_name
             refer_data.at[len(refer_data) - 1, args.dtype] = test_accuracy
         elif 'pass' not in refer_accuracy:
@@ -62,15 +62,15 @@
         timeout_models.append([model_name, test_accuracy])
         if refer_accuracy == "N/A":
             new_models.append([model_name, test_accuracy])
-            refer_data.loc[len(refer_data),:] = "N/A"
+            refer_data.loc[len(refer_data), :] = "N/A"
             refer_data.at[len(refer_data) - 1, "name"] = model_name
             refer_data.at[len(refer_data) - 1, args.dtype] = test_accuracy
     else:
         if refer_accuracy == "N/A":
             new_models.append([model_name, test_accuracy])
             # Not failed for new models
             expected_failed_models.append([model_name, test_accuracy])
-            refer_data.loc[len(refer_data),:] = "N/A"
+            refer_data.loc[len(refer_data), :] = "N/A"
             refer_data.at[len(refer_data) - 1, "name"] = model_name
             refer_data.at[len(refer_data) - 1, args.dtype] = test_accuracy
         elif "pass" in refer_accuracy:
@@ -81,7 +81,7 @@
                 refer_data.at[refer_row[0], args.dtype] = test_accuracy
 
 # pass rate
-print("============ Summary for {} {} {} accuracy ============".format(args.suite, args.dtype, args.mode))
+print(f"============ Summary for {args.suite} {args.dtype} {args.mode} accuracy ============")
 print("Total models:", len(model_names))
 print("Passed models:", len(passed_models))
 print("Real failed models:", len(real_failed_models), real_failed_models)
@@ -90,7 +90,7 @@
 print("New models:", len(new_models), new_models)
 print("Failed to passed models:", len(new_pass_models), new_pass_models)
 print("Not run/in models:", len(lost_models), lost_models)
-print("Pass rate: {:.2f}%".format(len(passed_models) / len(model_names) * 100))
+print(f"Pass rate: {len(passed_models) / len(model_names) * 100:.2f}%")
 
 if len(new_pass_models + new_models) > 0:
     print("NOTE: New models result, please update the reference", new_pass_models, new_models)

.github/ci_expected_accuracy/inductor_huggingface_inference.csv

@@ -1,47 +1,185 @@
 name,float32,bfloat16,float16,amp_bf16,amp_fp16
+
+
+
 AlbertForMaskedLM,pass,pass,pass,pass,pass
+
+
+
 AlbertForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 AllenaiLongformerBase,pass,pass,pass,pass,pass
+
+
+
 BartForCausalLM,pass,pass,pass,pass,pass
+
+
+
 BartForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 BertForMaskedLM,pass,pass,pass,pass,pass
+
+
+
 BertForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 BlenderbotForCausalLM,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
+
+
+
 BlenderbotSmallForCausalLM,pass,pass,pass,pass,pass
+
+
+
 BlenderbotSmallForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 CamemBert,pass,pass,pass,pass,pass
+
+
+
 DebertaForMaskedLM,pass,pass,pass,pass,pass
+
+
+
 DebertaForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 DebertaV2ForMaskedLM,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
+
+
+
 DebertaV2ForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 DistilBertForMaskedLM,pass,pass,pass,pass,pass
+
+
+
 DistilBertForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 DistillGPT2,pass,pass,pass,pass,pass
+
+
+
 ElectraForCausalLM,pass,pass,pass,pass,pass
+
+
+
 ElectraForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 GPT2ForSequenceClassification,pass,pass,pass,pass,pass
+
+
+
 GoogleFnet,pass,pass,pass,pass,pass
+
+
+
 LayoutLMForMaskedLM,pass,pass,pass,pass,pass
+
+
+
 LayoutLMForSequenceClassification,pass,pass,pass,pass,pass
+
+
+
 M2M100ForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 MBartForCausalLM,pass,pass,pass,pass,pass
+
+
+
 MBartForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 MT5ForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 MegatronBertForCausalLM,pass,pass,pass,pass,pass
+
+
+
 MegatronBertForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 MobileBertForMaskedLM,pass,pass,pass,pass,pass
+
+
+
 MobileBertForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 OPTForCausalLM,pass,pass,pass,pass,pass
+
+
+
 PLBartForCausalLM,pass,pass,pass,pass,pass
+
+
+
 PLBartForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 PegasusForCausalLM,pass,pass,pass,pass,pass
+
+
+
 PegasusForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 RobertaForCausalLM,pass,pass,pass,pass,pass
+
+
+
 RobertaForQuestionAnswering,pass,pass,pass,pass,pass
+
+
+
 Speech2Text2ForCausalLM,pass,pass,pass,pass,pass
+
+
+
 T5ForConditionalGeneration,pass,pass,pass,pass,pass
+
+
+
 T5Small,pass,pass,pass,pass,pass
+
+
+
 TrOCRForCausalLM,pass,pass,pass,pass,pass
+
+
+
 XGLMForCausalLM,pass,pass,pass,pass,pass
+
+
+
 XLNetLMHeadModel,pass,pass,pass,pass,pass
+
+
+
 YituTechConvBert,pass,pass,pass,pass,pass