update

bench_all.py

@@ -5,24 +5,24 @@
     seeds.append(random.randint(0, 100))
 
 seeds = [22, 92, 54, 86, 41]
-
+seeds = [22]
 config_paths = [
     "sample_configs/paper_image_cloud_configs.yaml",
     "sample_configs/paper_text_tabular_cloud_configs.yaml",
     "sample_configs/paper_text_cloud_configs.yaml",
-]
-frameworks = [
-    # "AutoGluon_best_master",
-    # "autokeras_master",
-    "ablation_base",
-    "ablation_greedy_soup",
-    "ablation_gradient_clip",
-    "ablation_warmup_steps",
-    # "ablation_cosine_decay",
-    # "ablation_weight_decay",
-    # "ablation_lr_decay",
-
-]
+
+frameworks = ['AutoGluon_best_master', 'ablation_base', 'ablation_add_greedy', 'ablation_add_grad_clip', 'ablation_add_warmup_steps', 'ablation_add_cosine_decay', 'ablation_add_weight_decay', 'ablation_add_lr_decay', 'AutoGluon_del_greedy', 'AutoGluon_del_grad_clip', 'AutoGluon_del_warmup_steps', 'AutoGluon_del_cosine_decay', 'AutoGluon_del_weight_decay', 'AutoGluon_del_lr_decay']
+
+#frameworks = [
+   # "ablation_base",
+   # "ablation_greedy_soup",
+   # "ablation_gradient_clip",
+   # "ablation_warmup_steps",
+   # "ablation_cosine_decay",
+   # "ablation_weight_decay",
+   # "ablation_lr_decay",
+#    "autokeras_master",
+#]
 constraints = [
     "g4_12x"
 ]

sample_configs/bench_all.py

@@ -0,0 +1,80 @@
+import random
+n_experiments = 5
+seeds = []
+for i in range(n_experiments):
+    seeds.append(random.randint(0, 100))
+
+seeds = [22, 92, 54, 86, 41]
+seeds = [22]
+config_paths = [
+    "sample_configs/paper_image_cloud_configs.yaml",
+    "sample_configs/paper_text_tabular_cloud_configs.yaml",
+    "sample_configs/paper_text_cloud_configs.yaml",
+<<<<<<< HEAD
+=======
+]
+frameworks = [
+    # "AutoGluon_best_master",
+    # "autokeras_master",
+    "ablation_base",
+    "ablation_greedy_soup",
+    "ablation_gradient_clip",
+    "ablation_warmup_steps",
+    # "ablation_cosine_decay",
+    # "ablation_weight_decay",
+    # "ablation_lr_decay",
+
+>>>>>>> 8bc504afe4a26167224ad4740a87db9844e25494
+]
+frameworks = ['AutoGluon_best_master', 'ablation_base', 'ablation_add_greedy', 'ablation_add_grad_clip', 'ablation_add_warmup_steps', 'ablation_add_cosine_decay', 'ablation_add_weight_decay', 'ablation_add_lr_decay', 'AutoGluon_del_greedy', 'AutoGluon_del_grad_clip', 'AutoGluon_del_warmup_steps', 'AutoGluon_del_cosine_decay', 'AutoGluon_del_weight_decay', 'AutoGluon_del_lr_decay']
+
+#frameworks = [
+   # "ablation_base",
+   # "ablation_greedy_soup",
+   # "ablation_gradient_clip",
+   # "ablation_warmup_steps",
+   # "ablation_cosine_decay",
+   # "ablation_weight_decay",
+   # "ablation_lr_decay",
+#    "autokeras_master",
+#]
+constraints = [
+    "g4_12x"
+]
+# module = "autokeras"
+module = "multimodal"
+
+import yaml
+import os
+import subprocess
+
+config_root = "./temp_configs"
+os.makedirs(config_root, exist_ok=True)
+
+for seed in seeds:
+    print("Seed: ", seed)
+    for constraint in constraints:
+        os.makedirs(f"{config_root}/{constraint}", exist_ok=True)
+        for framework in frameworks:
+            # for shot in fs:
+                config_dir = f"{config_root}/{constraint}/{framework}"
+                os.makedirs(config_dir, exist_ok=True)
+
+                for config_path in config_paths:
+                    with open(config_path, "r") as f:
+                        configs = yaml.safe_load(f)
+                        if constraint == "g4_12x":
+                            configs["cdk_context"]["PREFIX"] = f"{configs['cdk_context']['PREFIX']}-multi"
+                        configs["constraint"] = constraint
+                        configs["framework"] = framework
+                        configs["module"] = module
+                        configs["seed"] = seed 
+                        # configs["custom_dataloader"]["shot"] = shot
+                        configs["benchmark_name"] = f"{configs['benchmark_name']}-{seed}"
+                        new_config_path = os.path.join(config_dir, os.path.basename(config_path))
+                        with open(new_config_path, "w") as new_f:
+                            yaml.dump(configs, new_f)
+                        print("Running config: ", new_config_path)
+                        command = ["agbench", "run", new_config_path]
+                        subprocess.run(command)
+

sample_configs/dataloaders/paper_text_datasets.yaml

@@ -63,13 +63,12 @@ base: &base
 
 
 financial_news:
+  <<: *base
   url: s3://zs-models/datasets/financial_news/{lang}/{split}.csv
   splits:
     - train
   langs:
     - en
-  metric: accuracy
-  problem_type: classification
 
 MLDoc-11000:
   <<: *base