diff --git a/bench_all.py b/bench_all.py index 8fe1f5a8..94d7a0f3 100644 --- a/bench_all.py +++ b/bench_all.py @@ -5,24 +5,24 @@ seeds.append(random.randint(0, 100)) seeds = [22, 92, 54, 86, 41] - +seeds = [22] config_paths = [ "sample_configs/paper_image_cloud_configs.yaml", "sample_configs/paper_text_tabular_cloud_configs.yaml", "sample_configs/paper_text_cloud_configs.yaml", -] -frameworks = [ - # "AutoGluon_best_master", - # "autokeras_master", - "ablation_base", - "ablation_greedy_soup", - "ablation_gradient_clip", - "ablation_warmup_steps", - # "ablation_cosine_decay", - # "ablation_weight_decay", - # "ablation_lr_decay", - -] + +frameworks = ['AutoGluon_best_master', 'ablation_base', 'ablation_add_greedy', 'ablation_add_grad_clip', 'ablation_add_warmup_steps', 'ablation_add_cosine_decay', 'ablation_add_weight_decay', 'ablation_add_lr_decay', 'AutoGluon_del_greedy', 'AutoGluon_del_grad_clip', 'AutoGluon_del_warmup_steps', 'AutoGluon_del_cosine_decay', 'AutoGluon_del_weight_decay', 'AutoGluon_del_lr_decay'] + +#frameworks = [ + # "ablation_base", + # "ablation_greedy_soup", + # "ablation_gradient_clip", + # "ablation_warmup_steps", + # "ablation_cosine_decay", + # "ablation_weight_decay", + # "ablation_lr_decay", +# "autokeras_master", +#] constraints = [ "g4_12x" ] diff --git a/sample_configs/bench_all.py b/sample_configs/bench_all.py new file mode 100644 index 00000000..2dbf66c4 --- /dev/null +++ b/sample_configs/bench_all.py @@ -0,0 +1,80 @@ +import random +n_experiments = 5 +seeds = [] +for i in range(n_experiments): + seeds.append(random.randint(0, 100)) + +seeds = [22, 92, 54, 86, 41] +seeds = [22] +config_paths = [ + "sample_configs/paper_image_cloud_configs.yaml", + "sample_configs/paper_text_tabular_cloud_configs.yaml", + "sample_configs/paper_text_cloud_configs.yaml", +<<<<<<< HEAD +======= +] +frameworks = [ + # "AutoGluon_best_master", + # "autokeras_master", + "ablation_base", + "ablation_greedy_soup", + "ablation_gradient_clip", + "ablation_warmup_steps", + # "ablation_cosine_decay", + # "ablation_weight_decay", + # "ablation_lr_decay", + +>>>>>>> 8bc504afe4a26167224ad4740a87db9844e25494 +] +frameworks = ['AutoGluon_best_master', 'ablation_base', 'ablation_add_greedy', 'ablation_add_grad_clip', 'ablation_add_warmup_steps', 'ablation_add_cosine_decay', 'ablation_add_weight_decay', 'ablation_add_lr_decay', 'AutoGluon_del_greedy', 'AutoGluon_del_grad_clip', 'AutoGluon_del_warmup_steps', 'AutoGluon_del_cosine_decay', 'AutoGluon_del_weight_decay', 'AutoGluon_del_lr_decay'] + +#frameworks = [ + # "ablation_base", + # "ablation_greedy_soup", + # "ablation_gradient_clip", + # "ablation_warmup_steps", + # "ablation_cosine_decay", + # "ablation_weight_decay", + # "ablation_lr_decay", +# "autokeras_master", +#] +constraints = [ + "g4_12x" +] +# module = "autokeras" +module = "multimodal" + +import yaml +import os +import subprocess + +config_root = "./temp_configs" +os.makedirs(config_root, exist_ok=True) + +for seed in seeds: + print("Seed: ", seed) + for constraint in constraints: + os.makedirs(f"{config_root}/{constraint}", exist_ok=True) + for framework in frameworks: + # for shot in fs: + config_dir = f"{config_root}/{constraint}/{framework}" + os.makedirs(config_dir, exist_ok=True) + + for config_path in config_paths: + with open(config_path, "r") as f: + configs = yaml.safe_load(f) + if constraint == "g4_12x": + configs["cdk_context"]["PREFIX"] = f"{configs['cdk_context']['PREFIX']}-multi" + configs["constraint"] = constraint + configs["framework"] = framework + configs["module"] = module + configs["seed"] = seed + # configs["custom_dataloader"]["shot"] = shot + configs["benchmark_name"] = f"{configs['benchmark_name']}-{seed}" + new_config_path = os.path.join(config_dir, os.path.basename(config_path)) + with open(new_config_path, "w") as new_f: + yaml.dump(configs, new_f) + print("Running config: ", new_config_path) + command = ["agbench", "run", new_config_path] + subprocess.run(command) + diff --git a/sample_configs/dataloaders/paper_text_datasets.yaml b/sample_configs/dataloaders/paper_text_datasets.yaml index d3113109..067e12d1 100644 --- a/sample_configs/dataloaders/paper_text_datasets.yaml +++ b/sample_configs/dataloaders/paper_text_datasets.yaml @@ -63,13 +63,12 @@ base: &base financial_news: + <<: *base url: s3://zs-models/datasets/financial_news/{lang}/{split}.csv splits: - train langs: - en - metric: accuracy - problem_type: classification MLDoc-11000: <<: *base diff --git a/sample_configs/dataloaders/paper_text_tabular_datasets.yaml b/sample_configs/dataloaders/paper_text_tabular_datasets.yaml index b43fbab6..d2c9c79f 100644 --- a/sample_configs/dataloaders/paper_text_tabular_datasets.yaml +++ b/sample_configs/dataloaders/paper_text_tabular_datasets.yaml @@ -92,3 +92,281 @@ cal_house: - Sold Price metric: rmse problem_type: regression +base: &base + url: s3://automl-mm-bench/{name}/{split}.csv + test_split_name: test + splits: + - train + - test + feature_columns: + - ImageID + label_columns: + - LabelName + image_columns: + text_columns: + columns_to_drop: + metric: acc + problem_type: multiclass + + +prod: + <<: *base + url: s3://automl-mm-bench/machine_hack_product_sentiment/{split}.csv + test_split_name: dev + feature_columns: + - Product_Description + - Product_Type + label_columns: + - Sentiment + +airbnb: + <<: *base + url: s3://automl-mm-bench/airbnb_melbourne/{split}.pq + feature_columns: + null + label_columns: + - price_label + ignore_columns: + - id + - listing_url + - scrape_id + - last_scraped + - picture_url + - host_id + - host_url + - host_name + - host_thumbnail_url + - host_picture_url + - monthly_price + - weekly_price + - price + - calendar_last_scraped + +channel: + <<: *base + url: s3://automl-mm-bench/news_channel/{split}.csv + feature_columns: + null + label_columns: + - channel + ignore_columns: + null + +wine: + <<: *base + url: s3://automl-mm-bench/wine_reviews/{split}.csv + feature_columns: + null + label_columns: + - variety + ignore_columns: + null + +imdb: + <<: *base + url: s3://automl-mm-bench/imdb_genre_prediction/{split}.csv + feature_columns: + null + label_columns: + - Genre_is_Drama + ignore_columns: + null + metric: roc_auc + problem_type: binary + +jigsaw: + <<: *base + url: s3://automl-mm-bench/jigsaw_unintended_bias100K/{split}.pq + feature_columns: + - comment_text + - asian + - atheist + - bisexual + - black + - buddhist + - christian + - female + - heterosexual + - hindu + - homosexual_gay_or_lesbian + - intellectual_or_learning_disability + - jewish + - latino + - male + - muslim + - other_disability + - other_gender + - other_race_or_ethnicity + - other_religion + - other_sexual_orientation + - physical_disability + - psychiatric_or_mental_illness + - transgender + - white + - funny + - wow + - sad + - likes + - disagree + label_columns: + - target + metric: roc_auc + problem_type: binary + +fake: + <<: *base + url: s3://automl-mm-bench/fake_job_postings2/{split}.csv + feature_columns: + null + label_columns: + - fraudulent + ignore_columns: + null + metric: roc_auc + problem_type: binary + +kick: + <<: *base + url: s3://automl-mm-bench/kick_starter_funding/{split}.csv + feature_columns: + null + label_columns: + - final_status + ignore_columns: + null + metric: roc_auc + problem_type: binary + +ae: + <<: *base + url: s3://automl-mm-bench/ae_price_prediction/{split}.pq + feature_columns: + null + label_columns: + - price + ignore_columns: + - mrp + - pdp_url + metric: r2 + problem_type: regression + +qaa: + <<: *base + url: s3://automl-mm-bench/google_quest_qa/{split}.pq + test_split_name: dev + feature_columns: + - question_title + - question_body + - answer + - category + label_columns: + - answer_type_reason_explanation + metric: r2 + problem_type: regression + +qaq: + <<: *base + url: s3://automl-mm-bench/google_quest_qa/{split}.pq + test_split_name: dev + feature_columns: + - question_title + - question_body + - answer + - category + label_columns: + - question_type_reason_explanation + metric: r2 + problem_type: regression + +cloth: + <<: *base + url: s3://automl-mm-bench/women_clothing_review/{split}.pq + feature_columns: + - Title + - Review Text + - Age + - Division Name + - Department Name + - Class Name + label_columns: + - Rating + metric: r2 + problem_type: regression + +mercari: + <<: *base + url: s3://automl-mm-bench/mercari_price_suggestion100K/{split}.pq + feature_columns: + null + label_columns: + - log_price + ignore_columns: + - train_id + - price + metric: r2 + problem_type: regression + +jc: + <<: *base + url: s3://automl-mm-bench/jc_penney_products/{split}.csv + feature_columns: + null + label_columns: + - sale_price + ignore_columns: + null + metric: r2 + problem_type: regression + +pop: + <<: *base + url: s3://automl-mm-bench/news_popularity2/{split}.csv + feature_columns: + null + label_columns: + - log_shares + ignore_columns: + null + metric: r2 + problem_type: regression + +book: + <<: *base + url: s3://automl-mm-bench/machine_hack_competitions/predict_the_price_of_books/{split}.csv + feature_columns: + - Title + - Author + - Edition + - Reviews + - Ratings + - Synopsis + - Genre + - BookCategory + label_columns: + - Price + ignore_columns: + null + metric: r2 + problem_type: regression + +salary: + <<: *base + url: s3://automl-mm-bench/machine_hack_competitions/predict_the_data_scientists_salary_in_india_hackathon/{split}.csv + feature_columns: + null + label_columns: + - salary + ignore_columns: + null + metric: acc + problem_type: multiclass + +house: + <<: *base + url: s3://automl-mm-bench/kaggle-california-house-prices/{split}.csv + feature_columns: + null + label_columns: + - Sold Price + metric: r2 + problem_type: regression diff --git a/sample_configs/dataloaders/text_tabular_datasets.yaml b/sample_configs/dataloaders/text_tabular_datasets.yaml new file mode 100644 index 00000000..9f8e4d6c --- /dev/null +++ b/sample_configs/dataloaders/text_tabular_datasets.yaml @@ -0,0 +1,278 @@ +base: &base + url: s3://automl-mm-bench/{name}/{split}.csv + test_split_name: test + splits: + - train + - test + feature_columns: + - ImageID + label_columns: + - LabelName + image_columns: + text_columns: + columns_to_drop: + metric: acc + problem_type: multiclass + + +prod: + <<: *base + url: s3://automl-mm-bench/machine_hack_product_sentiment/{split}.csv + test_split_name: dev + feature_columns: + - Product_Description + - Product_Type + label_columns: + - Sentiment + +airbnb: + <<: *base + url: s3://automl-mm-bench/airbnb_melbourne/{split}.pq + feature_columns: + null + label_columns: + - price_label + ignore_columns: + - id + - listing_url + - scrape_id + - last_scraped + - picture_url + - host_id + - host_url + - host_name + - host_thumbnail_url + - host_picture_url + - monthly_price + - weekly_price + - price + - calendar_last_scraped + +channel: + <<: *base + url: s3://automl-mm-bench/news_channel/{split}.csv + feature_columns: + null + label_columns: + - channel + ignore_columns: + null + +wine: + <<: *base + url: s3://automl-mm-bench/wine_reviews/{split}.csv + feature_columns: + null + label_columns: + - variety + ignore_columns: + null + +imdb: + <<: *base + url: s3://automl-mm-bench/imdb_genre_prediction/{split}.csv + feature_columns: + null + label_columns: + - Genre_is_Drama + ignore_columns: + null + metric: roc_auc + problem_type: binary + +jigsaw: + <<: *base + url: s3://automl-mm-bench/jigsaw_unintended_bias100K/{split}.pq + feature_columns: + - comment_text + - asian + - atheist + - bisexual + - black + - buddhist + - christian + - female + - heterosexual + - hindu + - homosexual_gay_or_lesbian + - intellectual_or_learning_disability + - jewish + - latino + - male + - muslim + - other_disability + - other_gender + - other_race_or_ethnicity + - other_religion + - other_sexual_orientation + - physical_disability + - psychiatric_or_mental_illness + - transgender + - white + - funny + - wow + - sad + - likes + - disagree + label_columns: + - target + metric: roc_auc + problem_type: binary + +fake: + <<: *base + url: s3://automl-mm-bench/fake_job_postings2/{split}.csv + feature_columns: + null + label_columns: + - fraudulent + ignore_columns: + null + metric: roc_auc + problem_type: binary + +kick: + <<: *base + url: s3://automl-mm-bench/kick_starter_funding/{split}.csv + feature_columns: + null + label_columns: + - final_status + ignore_columns: + null + metric: roc_auc + problem_type: binary + +ae: + <<: *base + url: s3://automl-mm-bench/ae_price_prediction/{split}.pq + feature_columns: + null + label_columns: + - price + ignore_columns: + - mrp + - pdp_url + metric: r2 + problem_type: regression + +qaa: + <<: *base + url: s3://automl-mm-bench/google_quest_qa/{split}.pq + test_split_name: dev + feature_columns: + - question_title + - question_body + - answer + - category + label_columns: + - answer_type_reason_explanation + metric: r2 + problem_type: regression + +qaq: + <<: *base + url: s3://automl-mm-bench/google_quest_qa/{split}.pq + test_split_name: dev + feature_columns: + - question_title + - question_body + - answer + - category + label_columns: + - question_type_reason_explanation + metric: r2 + problem_type: regression + +cloth: + <<: *base + url: s3://automl-mm-bench/women_clothing_review/{split}.pq + feature_columns: + - Title + - Review Text + - Age + - Division Name + - Department Name + - Class Name + label_columns: + - Rating + metric: r2 + problem_type: regression + +mercari: + <<: *base + url: s3://automl-mm-bench/mercari_price_suggestion100K/{split}.pq + feature_columns: + null + label_columns: + - log_price + ignore_columns: + - train_id + - price + metric: r2 + problem_type: regression + +jc: + <<: *base + url: s3://automl-mm-bench/jc_penney_products/{split}.csv + feature_columns: + null + label_columns: + - sale_price + ignore_columns: + null + metric: r2 + problem_type: regression + +pop: + <<: *base + url: s3://automl-mm-bench/news_popularity2/{split}.csv + feature_columns: + null + label_columns: + - log_shares + ignore_columns: + null + metric: r2 + problem_type: regression + +book: + <<: *base + url: s3://automl-mm-bench/machine_hack_competitions/predict_the_price_of_books/{split}.csv + feature_columns: + - Title + - Author + - Edition + - Reviews + - Ratings + - Synopsis + - Genre + - BookCategory + label_columns: + - Price + ignore_columns: + null + metric: r2 + problem_type: regression + +salary: + <<: *base + url: s3://automl-mm-bench/machine_hack_competitions/predict_the_data_scientists_salary_in_india_hackathon/{split}.csv + feature_columns: + null + label_columns: + - salary + ignore_columns: + null + metric: acc + problem_type: multiclass + +house: + <<: *base + url: s3://automl-mm-bench/kaggle-california-house-prices/{split}.csv + feature_columns: + null + label_columns: + - Sold Price + metric: r2 + problem_type: regression diff --git a/sample_configs/paper_image_cloud_configs.yaml b/sample_configs/paper_image_cloud_configs.yaml index a3c5331f..904d3c1f 100644 --- a/sample_configs/paper_image_cloud_configs.yaml +++ b/sample_configs/paper_image_cloud_configs.yaml @@ -1,7 +1,7 @@ # Infra configurations cdk_context: # AWS infra configs used to setup AWS Batch environment with AWS CDK CDK_DEPLOY_ACCOUNT: 097403188315 # required, update with your AWS account - CDK_DEPLOY_REGION: us-east-1 # required, update with your desired region + CDK_DEPLOY_REGION: us-west-2 # required, update with your desired region PREFIX: ag-bench # Used to identify infra resources created, optional, default = ag-bench METRICS_BUCKET: autogluon-benchmark-metrics # required, has to be a globally unique name DATA_BUCKET: zs-models # optional, S3 bucket to download your private datasets diff --git a/sample_configs/paper_text_cloud_configs.yaml b/sample_configs/paper_text_cloud_configs.yaml index 2f98216d..a1173526 100644 --- a/sample_configs/paper_text_cloud_configs.yaml +++ b/sample_configs/paper_text_cloud_configs.yaml @@ -1,7 +1,7 @@ # Infra configurations cdk_context: # AWS infra configs used to setup AWS Batch environment with AWS CDK CDK_DEPLOY_ACCOUNT: 097403188315 # required, update with your AWS account - CDK_DEPLOY_REGION: us-east-1 # required, update with your desired region + CDK_DEPLOY_REGION: us-west-2 # required, update with your desired region PREFIX: ag-bench # Used to identify infra resources created, optional, default = ag-bench METRICS_BUCKET: autogluon-benchmark-metrics # required, has to be a globally unique name DATA_BUCKET: zs-models # optional, S3 bucket to download your private datasets diff --git a/sample_configs/paper_text_tabular_cloud_configs.yaml b/sample_configs/paper_text_tabular_cloud_configs.yaml index 16ba00c1..a42ad6f5 100644 --- a/sample_configs/paper_text_tabular_cloud_configs.yaml +++ b/sample_configs/paper_text_tabular_cloud_configs.yaml @@ -1,16 +1,16 @@ # Infra configurations cdk_context: # AWS infra configs used to setup AWS Batch environment with AWS CDK CDK_DEPLOY_ACCOUNT: 097403188315 # required, update with your AWS account - CDK_DEPLOY_REGION: us-east-1 # required, update with your desired region + CDK_DEPLOY_REGION: us-west-2 # required, update with your desired region PREFIX: ag-bench # Used to identify infra resources created, optional, default = ag-bench METRICS_BUCKET: autogluon-benchmark-metrics # required, has to be a globally unique name - DATA_BUCKET: automl-mm-bench # optional, S3 bucket to download your private datasets + DATA_BUCKET: zs-models # optional, S3 bucket to download your private datasets MAX_MACHINE_NUM: 1000 # optional, default 20 # BLOCK_DEVICE_VOLUME: 300 # optional, default 100GB # RESERVED_MEMORY_SIZE: 60000 # optional, default 15000MB # VPC_NAME: existing-vpc-name # optional -module: multimodal # required, choise of 'multimodal' pr 'autokeras' +module: autokeras # required, choise of 'multimodal' pr 'autokeras' mode: aws # required benchmark_name: ag_bench_text_tabular # required root_dir: ag_bench_runs # optional, default = "ag_bench_runs" @@ -27,6 +27,20 @@ framework: # framework options can be found at resources/multimodal_frameworks # ablation_weight_decay # ablation_lr_decay dataset_name: + # - prod + #- channel + #- wine + #- imdb + #- jigsaw + ##- fake + #- ae + #- qaa + #- qaq + #- mercari + #- jc + #- pop + #- book + #- salary - airbnb - kick_start - cloth_review diff --git a/sample_configs/resources/multimodal_frameworks.yaml b/sample_configs/resources/multimodal_frameworks.yaml index 623a6e9c..fd2e29e9 100644 --- a/sample_configs/resources/multimodal_frameworks.yaml +++ b/sample_configs/resources/multimodal_frameworks.yaml @@ -28,6 +28,140 @@ ablation_base: optimization.lr_decay: 1 +ablation_add_greedy: + repo: https://github.com/suzhoum/autogluon.git + version: add_constant_lr_decay + params: # MultimodalPredictor.fit(params) + presets: best_quality + hyperparameters: + optimization.gradient_clip_val: 0 + optimization.warmup_steps: 0 + optimization.lr_schedule: constant + optimization.weight_decay: 0 + optimization.lr_decay: 1 + + +ablation_add_grad_clip: + repo: https://github.com/suzhoum/autogluon.git + version: add_constant_lr_decay + params: # MultimodalPredictor.fit(params) + presets: best_quality + hyperparameters: + optimization.top_k_average_method: best + optimization.warmup_steps: 0 + optimization.lr_schedule: constant + optimization.weight_decay: 0 + optimization.lr_decay: 1 + + +ablation_add_warmup_steps: + repo: https://github.com/suzhoum/autogluon.git + version: add_constant_lr_decay + params: # MultimodalPredictor.fit(params) + presets: best_quality + hyperparameters: + optimization.top_k_average_method: best + optimization.gradient_clip_val: 0 + optimization.lr_schedule: constant + optimization.weight_decay: 0 + optimization.lr_decay: 1 + + + +ablation_add_cosine_decay: + repo: https://github.com/suzhoum/autogluon.git + version: add_constant_lr_decay + params: # MultimodalPredictor.fit(params) + presets: best_quality + hyperparameters: + optimization.top_k_average_method: best + optimization.gradient_clip_val: 0 + optimization.warmup_steps: 0 + optimization.weight_decay: 0 + optimization.lr_decay: 1 + + +ablation_add_weight_decay: + repo: https://github.com/suzhoum/autogluon.git + version: add_constant_lr_decay + params: # MultimodalPredictor.fit(params) + presets: best_quality + hyperparameters: + optimization.top_k_average_method: best + optimization.gradient_clip_val: 0 + optimization.warmup_steps: 0 + optimization.lr_schedule: constant + optimization.lr_decay: 1 + + +ablation_add_lr_decay: + repo: https://github.com/suzhoum/autogluon.git + version: add_constant_lr_decay + params: # MultimodalPredictor.fit(params) + presets: best_quality + hyperparameters: + optimization.top_k_average_method: best + optimization.gradient_clip_val: 0 + optimization.warmup_steps: 0 + optimization.lr_schedule: constant + optimization.weight_decay: 0 + + +AutoGluon_best_master: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + + +AutoGluon_del_greedy: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + optimization.top_k_average_method: best + +AutoGluon_del_grad_clip: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + optimization.gradient_clip_val: 0 + + +AutoGluon_del_warmup_steps: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + optimization.warmup_steps: 0 + + +AutoGluon_del_cosine_decay: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + optimization.lr_schedule: constant + +AutoGluon_del_weight_decay: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + optimization.weight_decay: 0 + + + +AutoGluon_del_lr_decay: + repo: https://github.com/autogluon/autogluon.git + version: master + params: # MultimodalPredictor.fit(params) + presets: best_quality + optimization.lr_decay: 1 + + + ablation_greedy_soup: repo: https://github.com/suzhoum/autogluon.git version: add_constant_lr_decay