From cebd43037e7dc523e548e7e43eb4bb86400525ca Mon Sep 17 00:00:00 2001 From: Ya-Lan Yang <63822845+ylyangtw@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:06:01 -0500 Subject: [PATCH 1/6] Update INCORE docs according to changes for Interdependent recovery (#422) * update * address feedbacks * update --- CHANGELOG.md | 5 ++++ .../analyses/housing_household_recovery.md | 28 +++++++++++-------- .../analyses/residential_building_recovery.md | 8 ++++-- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81cd878b..a7af1d43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to the INCORE documents generated by Sphinx package will be The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased + +### Changed +- Update docs according to support Interdependent recovery analysis [#417](https://github.com/IN-CORE/incore-docs/issues/417) + ## [4.13.0] - 2024-08-21 ### Changed diff --git a/manual_jb/content/analyses/housing_household_recovery.md b/manual_jb/content/analyses/housing_household_recovery.md index f7261d7d..d792f9b6 100644 --- a/manual_jb/content/analyses/housing_household_recovery.md +++ b/manual_jb/content/analyses/housing_household_recovery.md @@ -23,26 +23,30 @@ The output of the computation is the history of housing recovery changes for eac **Input parameters** -key name | type | name | description ---- | --- | --- | --- -`result_name` * | `str` | Result name | Name of the result dataset. -`seed` * | `int` | Seed | Initial value to seed the random number generator to ensure replication of the Markov Chain path'
in connection with Population Dislocation. -`t_delta` * | `float` | Time step | A size of the analysis time step. -`t_final` * | `float` | Time duration | Total duration. +key name | type | name | description +--- | --- |----------------| --- +`result_name` * | `str` | Result name | Name of the result dataset. +`seed` * | `int` | Seed | Initial value to seed the random number generator to ensure replication of the Markov Chain path'
in connection with Population Dislocation. +`t_delta` * | `float` | Time step | A size of the analysis time step. +`t_final` * | `float` | Time duration | Total duration. +`num_cpu` | `int` | Number of CPUs | If using parallel execution, the number of cpus to request. **Input datasets** -key name | type | name | description ---- | --- | --- | --- -`population_dislocation_block` * | [`incore:popDislocation`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:popDislocation) | Population dislocation | Population dislocation results. -`tpm` * | [`incore:houseRecTransitionProbMatrix`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:houseRecTransitionProbMatrix) | Probability matrix | A transition probability matrix that specifies
the corresponding Markov chain per social vulnerability level. -`initial_stage_probability` * | [`incore:houseRecInitialStageFactors`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:houseRecInitialStageFactors) | Mass probability | Initial mass probability function for stage 0 of the Markov Chain. +key name | type | name | description +--- | --- |-----------------------------| --- +`population_dislocation_block` * | [`incore:popDislocation`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:popDislocation) | Population dislocation | Population dislocation results. +`tpm` * | [`incore:houseRecTransitionProbMatrix`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:houseRecTransitionProbMatrix) | Probability matrix | A transition probability matrix that specifies
the corresponding Markov chain per social vulnerability level. +`initial_stage_probabilities` * | [`incore:houseRecInitialStageFactors`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:houseRecInitialStageFactors) | Mass probability | Initial mass probability function for stage 0 of the Markov Chain. +`sv_result` | [`incore:socialVulnerabilityScore`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:popDislocation) | Social vulnerability result | A csv file with zones containing demographic factors. +`zone_def_sv` | [`incore:zoneDefinitionsSocialVulnerability`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:houseRecTransitionProbMatrix) | Zone Definition: social vulnerability | A json file with thresholds and definitions per zone based on social vulnerability analysis. +`zone_def_hhinc` | [`incore:zoneDefinitionsHouseholdIncome`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:houseRecInitialStageFactors) | Zone Definition: household income | A json file with thresholds and definitions per zone based on household income. **Output datasets** key name | type | parent key | name | description --- | --- | --- | --- | --- -`result` * | [`incore:housingRecoveryHistory`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:housingRecoveryHistory) | `housing_recovery_block` | Results | A dataset containing results (format: CSV)
with housing recovery sequences at the individual household level. +`ds_result` * | [`incore:housingRecoveryHistory`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:housingRecoveryHistory) | `housing_recovery_block` | Results | A dataset containing results (format: CSV)
with housing recovery sequences at the individual household level. (* required) diff --git a/manual_jb/content/analyses/residential_building_recovery.md b/manual_jb/content/analyses/residential_building_recovery.md index 3c7e4e8f..08dbbc33 100644 --- a/manual_jb/content/analyses/residential_building_recovery.md +++ b/manual_jb/content/analyses/residential_building_recovery.md @@ -47,9 +47,11 @@ key name | type | name | description **Output datasets** -key name | type | parent key | name | description ---- | --- | --- | --- | --- -`residential_building_recovery` * | [`incore:buildingRecovery`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:buildingRecovery) | Results | A dataset containing results (format: CSV)
with percentages of residential building recovery. +key name | type | parent key | name | description +--- | --- | --- |------------------------| --- +`time_stepping_recovery` * | [`incore:buildingRecovery`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:buildingRecovery) | Results | Time Stepping Recovery | A dataset containing results (format: CSV)
with percentages of residential building recovery. +`total_delay` * | [`incore:buildingRecoveryDelay`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:buildingRecovery) | Results | Total Delay | A dataset containing results (format: CSV)
with delay time of residential building recovery. +`recovery` * | [`incore:buildingRecoveryTime`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:buildingRecovery) | Results | Recovery | A dataset containing results (format: CSV)
with delay time of residential building recovery. (* required) From 4233a471fad29e36c49c02d47f6dcd98394b5b9c Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Thu, 17 Oct 2024 15:56:04 -0500 Subject: [PATCH 2/6] Added project service API (#424) * Added project service API * changed service name --- CHANGELOG.md | 3 +++ restapi/index.html | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7af1d43..62a2f249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +### Added +- API documentation for project service [#423](https://github.com/IN-CORE/incore-docs/issues/423) + ### Changed - Update docs according to support Interdependent recovery analysis [#417](https://github.com/IN-CORE/incore-docs/issues/417) diff --git a/restapi/index.html b/restapi/index.html index 59c32a08..7c503f74 100644 --- a/restapi/index.html +++ b/restapi/index.html @@ -43,7 +43,8 @@ {url: window.location.protocol + '//' + window.location.host + "/data/api/openapi.json", name: "Data"}, {url: window.location.protocol + '//' + window.location.host + "/dfr3/api/openapi.json", name: "DFR3"}, {url: window.location.protocol + '//' + window.location.host + "/semantics/api/openapi.json", name: "Semantics"}, - {url: window.location.protocol + '//' + window.location.host + "/space/api/openapi.json", name: "Space"} + {url: window.location.protocol + '//' + window.location.host + "/space/api/openapi.json", name: "Space"}, + {url: window.location.protocol + '//' + window.location.host + "/project/api/openapi.json", name: "Project"} ], dom_id: '#swagger-ui', deepLinking: true, From 023d042921fcca5c1d2df74f0c2ba7c5c276b487 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 21 Oct 2024 10:58:36 -0500 Subject: [PATCH 3/6] 425 add doc and notebook for equity metric analysis (#426) * markdowns * add fully chained equity metric notebook * finalize the notebook * remove analysis in the title * remove reference to dev service * update the documentation per request --- CHANGELOG.md | 1 + manual_jb/content/_toc.yml | 1 + manual_jb/content/analyses.md | 65 ++-- manual_jb/content/analyses/equity_metric.md | 84 ++++ notebooks/equity_metric.ipynb | 403 ++++++++++++++++++++ notebooks/housing_household_recovery.ipynb | 7 +- 6 files changed, 528 insertions(+), 33 deletions(-) create mode 100644 manual_jb/content/analyses/equity_metric.md create mode 100644 notebooks/equity_metric.ipynb diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a2f249..952755aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - API documentation for project service [#423](https://github.com/IN-CORE/incore-docs/issues/423) +- Documentation for Equity Metric Analysis [#425](https://github.com/IN-CORE/incore-docs/issues/425) ### Changed - Update docs according to support Interdependent recovery analysis [#417](https://github.com/IN-CORE/incore-docs/issues/417) diff --git a/manual_jb/content/_toc.yml b/manual_jb/content/_toc.yml index d87649bc..e1fa269e 100644 --- a/manual_jb/content/_toc.yml +++ b/manual_jb/content/_toc.yml @@ -31,6 +31,7 @@ chapters: - file: analyses/epf_repair_cost - file: analyses/epf_restoration - file: analyses/epn_functionality + - file: analyses/equity_metric - file: analyses/galveston_cge - file: analyses/gas_facility_damage - file: analyses/housing_household_recovery diff --git a/manual_jb/content/analyses.md b/manual_jb/content/analyses.md index af62ba5f..956b4242 100644 --- a/manual_jb/content/analyses.md +++ b/manual_jb/content/analyses.md @@ -17,35 +17,36 @@ 15. [Electric power facility repair cost](analyses/epf_repair_cost) 16. [Electric power facility restoration](analyses/epf_restoration) 17. [Electric power network functionality](analyses/epn_functionality) -18. [Galveston Computable General Equilibrium (CGE)](analyses/galveston_cge.md) -19. [Gas facility damage](analyses/gas_facility_damage) -20. [Household-level housing sequential recovery](analyses/housing_household_recovery) -21. [Housing recovery](analyses/housing_recovery) -22. [Housing unit allocation](analyses/housingunitallocation) -23. [Interdependent Network Design Problem](analyses/indp) -24. [Joplin Computable General Equilibrium (CGE)](analyses/joplin_cge) -25. [Joplin empirical building restoration](analyses/joplin_empirical_building_restoration) -26. [Machine Learning Enabled Computable General Equilibrium (CGE) - Salt Lake City](analyses/ml_slc_cge.md) -27. [Mean damage](analyses/mean_dmg) -28. [Monte Carlo limit state probability](analyses/mc_limit_state_prob) -29. [Multi-objective retrofit optimization](analyses/multi_retrofit_optimization) -30. [Network cascading interdependency functionality](analyses/nci_functionality) -31. [Pipeline damage](analyses/pipeline_dmg) -32. [Pipeline damage with repair rate](analyses/pipeline_dmg_w_repair_rate) -33. [Pipeline functionality](analyses/pipeline_functionality) -34. [Pipeline repair cost](analyses/pipeline_repair_cost) -35. [Pipeline restoration](analyses/pipeline_restoration) -36. [Population dislocation](analyses/populationdislocation) -37. [Residential building recovery](analyses/residential_building_recovery) -38. [Road damage](analyses/road_dmg) -39. [Salt Lake City Computable General Equilibrium (CGE)](analyses/slc_cge.md) -40. [Seaside Computable General Equilibrium (CGE)](analyses/seaside_cge) -41. [Social Vulnerability](analyses/social_vulnerability) -42. [Social Vulnerability Score](analyses/social_vulnerability_score) -43. [Tornado electric power network (EPN) damage](analyses/tornadoepn_dmg) -44. [Traffic flow recovery](analyses/traffic_flow_recovery) -45. [Transportation recovery](analyses/transportation_recovery) -46. [Water facility damage](analyses/waterfacility_dmg) -47. [Water network functionality](analyses/wfn_functionality) -48. [Water facility repair cost](analyses/water_facility_repair_cost) -49. [Water facility restoration](analyses/water_facility_restoration) +18. [Equity Metric](analyses/equity_metric) +19. [Galveston Computable General Equilibrium (CGE)](analyses/galveston_cge.md) +20. [Gas facility damage](analyses/gas_facility_damage) +21. [Household-level housing sequential recovery](analyses/housing_household_recovery) +22. [Housing recovery](analyses/housing_recovery) +23. [Housing unit allocation](analyses/housingunitallocation) +24. [Interdependent Network Design Problem](analyses/indp) +25. [Joplin Computable General Equilibrium (CGE)](analyses/joplin_cge) +26. [Joplin empirical building restoration](analyses/joplin_empirical_building_restoration) +27. [Machine Learning Enabled Computable General Equilibrium (CGE) - Salt Lake City](analyses/ml_slc_cge.md) +28. [Mean damage](analyses/mean_dmg) +29. [Monte Carlo limit state probability](analyses/mc_limit_state_prob) +30. [Multi-objective retrofit optimization](analyses/multi_retrofit_optimization) +31. [Network cascading interdependency functionality](analyses/nci_functionality) +32. [Pipeline damage](analyses/pipeline_dmg) +33. [Pipeline damage with repair rate](analyses/pipeline_dmg_w_repair_rate) +34. [Pipeline functionality](analyses/pipeline_functionality) +35. [Pipeline repair cost](analyses/pipeline_repair_cost) +36. [Pipeline restoration](analyses/pipeline_restoration) +37. [Population dislocation](analyses/populationdislocation) +38. [Residential building recovery](analyses/residential_building_recovery) +39. [Road damage](analyses/road_dmg) +40. [Salt Lake City Computable General Equilibrium (CGE)](analyses/slc_cge.md) +41. [Seaside Computable General Equilibrium (CGE)](analyses/seaside_cge) +42. [Social Vulnerability](analyses/social_vulnerability) +43. [Social Vulnerability Score](analyses/social_vulnerability_score) +44. [Tornado electric power network (EPN) damage](analyses/tornadoepn_dmg) +45. [Traffic flow recovery](analyses/traffic_flow_recovery) +46. [Transportation recovery](analyses/transportation_recovery) +47. [Water facility damage](analyses/waterfacility_dmg) +48. [Water network functionality](analyses/wfn_functionality) +49. [Water facility repair cost](analyses/water_facility_repair_cost) +50. [Water facility restoration](analyses/water_facility_restoration) diff --git a/manual_jb/content/analyses/equity_metric.md b/manual_jb/content/analyses/equity_metric.md new file mode 100644 index 00000000..12363925 --- /dev/null +++ b/manual_jb/content/analyses/equity_metric.md @@ -0,0 +1,84 @@ +# Equity Metric + +**Description** + +The algorithm computes equity metrics to characterize the inequity present in the infrastructure service provision +between two groups of concerns’ (e.g., low income vs. non-low income, minority vs non-minority, etc.). The metrics are +built upon Theil’s T, which is a common metric to compute the inequality present in the dispersion of a scarce +resource (e.g., income). The metrics could also be implemented to assess inequity in different scenarios with other +scarce resources of concern. + +To compute the metrics, a scarce resource should be formulated and calculated for an infrastructure application. The +scarce resource for infrastructure can be taken as any user defined relevant values, such as a resilience score (i.e., +probability of service provision) or recovery time. We've included a companion utility class where we define and +prepare recovery time as a scarce resource. Other scarce resources can and could be explored if provided by the user. + +The equity metric allows for an equity assessment of the current infrastructure service provision. It also enables the +assessment of equity gains for a given retrofit plan and can be integrated into an overall decision-making process.. +The output metrics tell the following information 1) Theil's T - overall amount of inequality in scarce resource's +dispersion across a community (distributional inequity) 2) Between Zone Inequality (BZI) - amount of inequality +attributed to scarce resource differences between groups (restorative inequity), and 3) Within Zone Inequality (WZI) - +amount of inequality due to resource differences among singular groups. + +**Contributors** + +- Science: Abigail L. Beck, Ph.D, Eun Jeong Cha, Ph.D, Walter Peacock, Ph.D +- Implementation: NCSA IN-CORE Dev Team + +**Related publications** + +- Beck, A.L., Cha, E.J. & Peacock, W.G. "Incorporation of Equity into Infrastructure Decision-Making: Development of an + Equity Metric for Infrastructure Retrofitting," The 14th International Conference on Applications of Statistics and + Probability in Civil Engineering (ICASP14), Dublin, Ireland, July, 2023. http://hdl.handle.net/2262/103309 + +**Input parameters** + + key name | type | name | description +-----------------------------------------|-------|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + `result_name` * | `str` | Result name | Name of the result dataset. + `division_decision_column` * | `str` | Division decision column name | Column name of a binary variable associated with each household used to group it into two groups (e.g. low income vs non low income, minority vs non-minority, social vulnerability) + +**Input datasets** + + key name | type | name | description +----------------------------------------|---------------------------------------------------------------------------------------------------------------------|--------------------|------------------------------------------------------------------------ + `housing_unit_allocation` * | [`incore:housingUnitAllocation`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:housingUnitAllocation) | Housing allocation | A housing unit allocation dataset. + `scarce_resource` * | [`incore:scarceResource`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:scarceResource) | Scarce resource | Scarce resource dataset e.g. probability of service, return time, etc. + +**Output datasets** + + key name | type | parent key | name | description +------------------------------|---------------------------------------------------------------------------------------------------|------------|---------------|------------------------------------------------------------------------------------------------------- + `equity_metric` * | [`incore:equityMetric`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:equityMetric) | | Equity Metric | CSV file of equity metric, including Theil’s T Value, Between Zone Inequality, Within Zone Inequality + +(* required) + +**Execution** + +code snippet: + +``` + client = IncoreClient() + datasvc = DataService(client) + + # Example of preparing scarce resource + repair_time_df = Dataset.from_data_service(housing_recovery_id, datasvc).get_dataframe_from_csv() + scarce_resource_df = EquityMetricUtil.prepare_return_time_as_scarce_resource( + repair_time_df + ) + scarce_resource = Dataset.from_dataframe( + scarce_resource_df, "scarce_resource", data_type="incore:scarceResource" + ) + + # Example of running equity metric analysis + equity_metric = EquityMetric(client) + equity_metric.set_parameter("result_name", "Galveston_recovery_time") + equity_metric.set_parameter("division_decision_column", "SVI") + equity_metric.load_remote_input_dataset( + "housing_unit_allocation", housing_unit_allocation_id + ) + equity_metric.set_input_dataset("scarce_resource", scarce_resource) + equity_metric.run_analysis() +``` + +full analysis: [equity_metric.ipynb](https://github.com/IN-CORE/incore-docs/blob/main/notebooks/equity_metric.ipynb) diff --git a/notebooks/equity_metric.ipynb b/notebooks/equity_metric.ipynb new file mode 100644 index 00000000..9a9df2c4 --- /dev/null +++ b/notebooks/equity_metric.ipynb @@ -0,0 +1,403 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b91f401e-91da-402c-bed4-2801b00a59d2", + "metadata": {}, + "outputs": [], + "source": [ + "from pyincore import IncoreClient, Dataset, DataService, FragilityService, MappingSet\n", + "from pyincore.analyses.equitymetric import EquityMetric, EquityMetricUtil\n", + "from pyincore.analyses.buildingstructuraldamage import BuildingStructuralDamage\n", + "from pyincore.analyses.socialvulnerabilityscore import SocialVulnerabilityScore\n", + "from pyincore.analyses.housingrecoverysequential import HousingRecoverySequential\n", + "from pyincore.analyses.buildingnonstructuraldamage import BuildingNonStructDamage\n", + "from pyincore.analyses.combinedwindwavesurgebuildingdamage import CombinedWindWaveSurgeBuildingDamage\n", + "from pyincore.analyses.populationdislocation import PopulationDislocation\n", + "\n", + "from pyincore_data.censusutil import CensusUtil" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af21ffc2-1de0-40bd-8426-20c5eb8a43c4", + "metadata": {}, + "outputs": [], + "source": [ + "client = IncoreClient()\n", + "datasvc = DataService(client)" + ] + }, + { + "cell_type": "markdown", + "id": "67641f60-6346-42e3-9814-77544c889441", + "metadata": {}, + "source": [ + "## Prepare recovery time as scarce resource" + ] + }, + { + "cell_type": "markdown", + "id": "302f229c-ce0a-41f6-9ece-fed4b9c293da", + "metadata": {}, + "source": [ + "### 1. Galveston Combined damage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfa3daf8-71e6-4e38-b60c-597b7fdfaea8", + "metadata": {}, + "outputs": [], + "source": [ + "hazard_type = \"hurricane\"\n", + "\n", + "# Galveston Hurricane Ike\n", + "hazard_id = \"5fa5a228b6429615aeea4410\"\n", + "\n", + "# Galveston inventory data\n", + "# bldg_dataset_id = \"63053ddaf5438e1f8c517fed\" # Full city\n", + "bldg_dataset_id = \"63ff6b135c35c0353d5ed3ac\" # Island Only" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f09018-eb7f-4b69-b3c7-ff67d8874e82", + "metadata": {}, + "outputs": [], + "source": [ + "# Building Wind Fragility mapping\n", + "mapping_id = \"62fef3a6cef2881193f2261d\"\n", + "fragility_service = FragilityService(client)\n", + "mapping_set = MappingSet(fragility_service.get_mapping(mapping_id))\n", + "# wind building damage\n", + "w_bldg_dmg = BuildingStructuralDamage(client)\n", + "w_bldg_dmg.load_remote_input_dataset(\"buildings\", bldg_dataset_id)\n", + "w_bldg_dmg.set_input_dataset(\"dfr3_mapping_set\", mapping_set)\n", + "w_bldg_dmg.set_parameter(\"result_name\", \"Galveston-wind-dmg\")\n", + "w_bldg_dmg.set_parameter(\"hazard_type\", hazard_type)\n", + "w_bldg_dmg.set_parameter(\"hazard_id\", hazard_id)\n", + "w_bldg_dmg.set_parameter(\"num_cpu\", 8)\n", + "w_bldg_dmg.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa158ebd-cde4-4faa-889b-5ce5b29fd543", + "metadata": {}, + "outputs": [], + "source": [ + "# Surge-wave mapping\n", + "mapping_id = \"6303e51bd76c6d0e1f6be080\"\n", + "mapping_set = MappingSet(fragility_service.get_mapping(mapping_id))\n", + "# surge-wave building damage\n", + "sw_bldg_dmg = BuildingStructuralDamage(client)\n", + "sw_bldg_dmg.load_remote_input_dataset(\"buildings\", bldg_dataset_id)\n", + "sw_bldg_dmg.set_input_dataset(\"dfr3_mapping_set\", mapping_set)\n", + "sw_bldg_dmg.set_parameter(\"result_name\", \"Galveston-sw-dmg\")\n", + "sw_bldg_dmg.set_parameter(\"hazard_type\", hazard_type)\n", + "sw_bldg_dmg.set_parameter(\"hazard_id\", hazard_id)\n", + "sw_bldg_dmg.set_parameter(\"num_cpu\", 8)\n", + "sw_bldg_dmg.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930978ad-64cb-4c30-97a5-1c74bee464c6", + "metadata": {}, + "outputs": [], + "source": [ + "# flood mapping\n", + "mapping_id = \"62fefd688a30d30dac57bbd7\"\n", + "mapping_set = MappingSet(fragility_service.get_mapping(mapping_id))\n", + "# flood building damage\n", + "f_bldg_dmg = BuildingNonStructDamage(client)\n", + "f_bldg_dmg.load_remote_input_dataset(\"buildings\", bldg_dataset_id)\n", + "f_bldg_dmg.set_input_dataset(\"dfr3_mapping_set\", mapping_set)\n", + "f_bldg_dmg.set_parameter(\"result_name\", \"Galveston-flood-dmg\")\n", + "f_bldg_dmg.set_parameter(\"fragility_key\", \"Non-Retrofit Fragility ID Code\")\n", + "f_bldg_dmg.set_parameter(\"hazard_type\", hazard_type)\n", + "f_bldg_dmg.set_parameter(\"hazard_id\", hazard_id)\n", + "f_bldg_dmg.set_parameter(\"num_cpu\", 8)\n", + "f_bldg_dmg.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9eb14ccd-64b5-49b2-9043-5c35e999496d", + "metadata": {}, + "outputs": [], + "source": [ + "surge_wave_damage = sw_bldg_dmg.get_output_dataset(\"ds_result\")\n", + "wind_damage = w_bldg_dmg.get_output_dataset(\"ds_result\")\n", + "flood_damage = f_bldg_dmg.get_output_dataset(\"result\")\n", + "combined_bldg_dmg = CombinedWindWaveSurgeBuildingDamage(client)\n", + "result_name = \"Galveston-combined-dmg\"\n", + "combined_bldg_dmg.set_input_dataset(\"surge_wave_damage\", surge_wave_damage)\n", + "combined_bldg_dmg.set_input_dataset(\"wind_damage\", wind_damage)\n", + "combined_bldg_dmg.set_input_dataset(\"flood_damage\", flood_damage)\n", + "combined_bldg_dmg.set_parameter(\"result_name\", result_name)\n", + "combined_bldg_dmg.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19e3c476-b317-4057-be56-f6096cd8ad99", + "metadata": {}, + "outputs": [], + "source": [ + "combined_dmg = combined_bldg_dmg.get_output_dataset(\"ds_result\")\n", + "combined_dmg_df = combined_dmg.get_dataframe_from_csv(low_memory=False)" + ] + }, + { + "cell_type": "markdown", + "id": "91bb051a-b5ee-4cbf-a7da-28b3eaf65f81", + "metadata": {}, + "source": [ + "### 2. Population Dislocation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e87c801a-fca3-4911-98e4-d7f9c025096f", + "metadata": {}, + "outputs": [], + "source": [ + "pop_dis = PopulationDislocation(client)\n", + "value_loss = \"60354810e379f22e16560dbd\"\n", + "bg_data = \"603545f2dcda03378087e708\"\n", + "hua_result_id = \"63ee615d62b9d001e6a2b660\" # city\n", + "# hua_result_id = \"63ff8e895367c2261b4cb2ef\" # island\n", + "\n", + "pop_dis.load_remote_input_dataset(\"block_group_data\", bg_data)\n", + "pop_dis.load_remote_input_dataset(\"value_loss_param\", value_loss)\n", + "pop_dis.load_remote_input_dataset(\"housing_unit_allocation\", hua_result_id)\n", + "\n", + "pop_dis.set_input_dataset(\"building_dmg\", combined_dmg)\n", + "\n", + "result_name = \"galveston-pop-disl-results\"\n", + "seed = 1111\n", + "\n", + "pop_dis.set_parameter(\"result_name\", result_name)\n", + "pop_dis.set_parameter(\"seed\", seed)\n", + "pop_dis.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1564962e-0921-4fe2-b388-ecae777de6b3", + "metadata": {}, + "outputs": [], + "source": [ + "population_dislocation_result = pop_dis.get_output_dataset(\"result\")\n", + "population_dislocation_result_df = population_dislocation_result.get_dataframe_from_csv(\n", + " low_memory=False\n", + ")\n", + "population_dislocation_result_df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "aeb4c1a2-4095-42eb-a733-d93b14500455", + "metadata": {}, + "source": [ + "### 3. Run social vulnerability score analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24414d12-1772-4cb3-80d8-3c936c88455a", + "metadata": {}, + "outputs": [], + "source": [ + "# get fips code to use fetch census data\n", + "fips = CensusUtil.get_fips_by_state_county(state=\"texas\", county=\"galveston\")\n", + "state_code = fips[:2]\n", + "county_code = fips[2:]\n", + "\n", + "navs = CensusUtil.national_ave_values(year=2020)\n", + "national_vulnerability_feature_averages = Dataset.from_csv_data(navs, name=\"national_vulnerability_feature_averages.csv\",\n", + " data_type=\"incore:socialVulnerabilityFeatureAverages\")\n", + "\n", + "geo_type = \"block%20group:*\"\n", + "# geo_type = \"tract:*\"\n", + "social_vulnerability_dem_factors_df = CensusUtil.demographic_factors(state_code, county_code, year=2020,\n", + " geo_type=geo_type)\n", + "\n", + "# Temp fix: remove bad data point\n", + "social_vulnerability_dem_factors_df = social_vulnerability_dem_factors_df.dropna()\n", + "\n", + "social_vulnerability_dem_factors = Dataset.from_dataframe(social_vulnerability_dem_factors_df,\n", + " name=\"social_vunlnerability_dem_factors\",\n", + " data_type=\"incore:socialVulnerabilityDemFactors\")\n", + "\n", + "# Configure pyincore access and the analysis with the information above\n", + "social_vulnerability = SocialVulnerabilityScore(client)\n", + "\n", + "social_vulnerability.set_parameter(\"result_name\", \"social_vulnerabilty\")\n", + "social_vulnerability.set_input_dataset(\"national_vulnerability_feature_averages\", national_vulnerability_feature_averages)\n", + "social_vulnerability.set_input_dataset(\"social_vulnerability_demographic_factors\", social_vulnerability_dem_factors)\n", + "\n", + "# Run social vulnerability damage analysis\n", + "result = social_vulnerability.run_analysis()\n", + "sv_result = social_vulnerability.get_output_dataset(\"sv_result\")" + ] + }, + { + "cell_type": "markdown", + "id": "9bfb49a3-31ec-4840-b704-7970d26af31a", + "metadata": {}, + "source": [ + "### 4. Housing Household recovery analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8edeea9e-51f4-4808-a5c2-cd99f817fd96", + "metadata": {}, + "outputs": [], + "source": [ + "# Population dislocation result aggregated to the block group level.\n", + "population_dislocation = \"60f5ea5d544e944c3cec66dd\" # Galveston testbed\n", + "# Transition probability matrix per social vulnerability level.\n", + "transition_probability_matrix = \"60f5e2ae544e944c3cec0794\"\n", + "# Initial mass probability function for stage 0.\n", + "initial_probability_vector = \"60f5e918544e944c3cec668b\"\n", + "\n", + "# Create housing recovery instance\n", + "housing_recovery = HousingRecoverySequential(client)\n", + "\n", + "# Load input datasets\n", + "housing_recovery.set_input_dataset(\"population_dislocation_block\", population_dislocation_result)\n", + "housing_recovery.load_remote_input_dataset(\"tpm\", transition_probability_matrix)\n", + "housing_recovery.load_remote_input_dataset(\"initial_stage_probabilities\", initial_probability_vector)\n", + "\n", + "# Chain with Social Vulnerability output\n", + "housing_recovery.set_input_dataset('sv_result', sv_result)\n", + "\n", + "# Set analysis parameters\n", + "housing_recovery.set_parameter(\"result_name\", \"housing_recovery_result\")\n", + "housing_recovery.set_parameter(\"seed\", 1234)\n", + "housing_recovery.set_parameter(\"t_delta\", 1.0)\n", + "housing_recovery.set_parameter(\"t_final\", 90.0)\n", + "\n", + "housing_recovery.run()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3aa72827-77d8-4ab0-9b93-f59dd700daac", + "metadata": {}, + "outputs": [], + "source": [ + "# Retrieve result dataset\n", + "hhrs_result = housing_recovery.get_output_dataset(\"ds_result\")\n", + "recovery_df = hhrs_result.get_dataframe_from_csv()" + ] + }, + { + "cell_type": "markdown", + "id": "01f8ab66-42c2-479c-8832-960c7f569f0a", + "metadata": {}, + "source": [ + "### 5. Prepare recovery time as scarce resource" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16717065-6fb0-4c78-93eb-1d697a648f28", + "metadata": {}, + "outputs": [], + "source": [ + "scarce_resource_df = EquityMetricUtil.prepare_return_time_as_scarce_resource(\n", + " recovery_df\n", + ")\n", + "scarce_resource = Dataset.from_dataframe(\n", + " scarce_resource_df, \"scarce_resource\", data_type=\"incore:scarceResource\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "367c9293-9b94-4354-8f85-e4a513cbbcd1", + "metadata": {}, + "outputs": [], + "source": [ + "scarce_resource_df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "993619eb-1735-4b0c-8b2e-b4fd90b5e1ac", + "metadata": {}, + "source": [ + "## Run Equity Metric Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95390162-fcf8-4f77-b66c-a7f01a3ea1eb", + "metadata": {}, + "outputs": [], + "source": [ + "equity_metric = EquityMetric(client)\n", + "equity_metric.set_parameter(\"result_name\", \"Galveston_recovery_time\")\n", + "equity_metric.set_parameter(\"division_decision_column\", \"SVI\")\n", + "equity_metric.load_remote_input_dataset(\n", + " \"housing_unit_allocation\", \"63ee615d62b9d001e6a2b660\"\n", + ")\n", + "equity_metric.set_input_dataset(\"scarce_resource\", scarce_resource)\n", + "em_result = equity_metric.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b057669e-075d-4491-82c3-810fdae12a67", + "metadata": {}, + "outputs": [], + "source": [ + "equity_metric_df = equity_metric.get_output_dataset(\"equity_metric\").get_dataframe_from_csv()\n", + "equity_metric_df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/housing_household_recovery.ipynb b/notebooks/housing_household_recovery.ipynb index 25627c5f..ea5713b3 100644 --- a/notebooks/housing_household_recovery.ipynb +++ b/notebooks/housing_household_recovery.ipynb @@ -4,6 +4,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": false, "jupyter": { "outputs_hidden": false }, @@ -25,6 +26,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": false, "jupyter": { "outputs_hidden": false }, @@ -182,6 +184,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": false, "jupyter": { "outputs_hidden": false }, @@ -228,6 +231,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": false, "jupyter": { "outputs_hidden": false }, @@ -251,6 +255,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": false, "jupyter": { "outputs_hidden": false }, @@ -303,7 +308,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.9.19" } }, "nbformat": 4, From e52fa83a7dcd50bbb5a2c79540ecdadb107bcae0 Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Tue, 22 Oct 2024 08:50:52 -0500 Subject: [PATCH 4/6] Release 4.14.0 --- CHANGELOG.md | 2 +- manual_jb/content/_config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 952755aa..d3081650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the INCORE documents generated by Sphinx package will be The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## Unreleased +## [4.14.0] - 2024-10-23 ### Added - API documentation for project service [#423](https://github.com/IN-CORE/incore-docs/issues/423) diff --git a/manual_jb/content/_config.yml b/manual_jb/content/_config.yml index 754ebfbc..58e1e1c8 100644 --- a/manual_jb/content/_config.yml +++ b/manual_jb/content/_config.yml @@ -3,7 +3,7 @@ project: "IN-CORE Manual" author: IN-CORE Community copyright: "2023" # Copyright year to be placed in the footer logo: images/resilience-logo.png -version: "4.13.0" # Version is not part of JB config, we use it for autobuild of incore-docs +version: "4.14.0" # Version is not part of JB config, we use it for autobuild of incore-docs exclude_patterns: [_build, .DS_Store, "**.ipynb_checkpoints"] repository: From f6369ff13c5af93d7ec0f990352efad6da03c747 Mon Sep 17 00:00:00 2001 From: Rashmil Panchani <32737711+Rashmil-1999@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:35:21 -0500 Subject: [PATCH 5/6] add ml enabled joplin cge notebook (#430) * add ml enabled joplin cge notebook * add entried in mardown files * add entry in toc file --- CHANGELOG.md | 1 + manual_jb/content/_toc.yml | 2 + manual_jb/content/analyses.md | 49 +- manual_jb/content/analyses/ml_joplin_cge.md | 61 ++ notebooks/ml_enabled_joplin_cge.ipynb | 740 ++++++++++++++++++++ 5 files changed, 829 insertions(+), 24 deletions(-) create mode 100644 manual_jb/content/analyses/ml_joplin_cge.md create mode 100644 notebooks/ml_enabled_joplin_cge.ipynb diff --git a/CHANGELOG.md b/CHANGELOG.md index d3081650..f79de28d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - API documentation for project service [#423](https://github.com/IN-CORE/incore-docs/issues/423) - Documentation for Equity Metric Analysis [#425](https://github.com/IN-CORE/incore-docs/issues/425) +- ML Enabled Joplin CGE notebook [#429](https://github.com/IN-CORE/incore-docs/issues/429) ### Changed - Update docs according to support Interdependent recovery analysis [#417](https://github.com/IN-CORE/incore-docs/issues/417) diff --git a/manual_jb/content/_toc.yml b/manual_jb/content/_toc.yml index e1fa269e..7a55ac71 100644 --- a/manual_jb/content/_toc.yml +++ b/manual_jb/content/_toc.yml @@ -42,6 +42,8 @@ chapters: - file: analyses/joplin_empirical_restoration - file: analyses/mean_dmg - file: analyses/mc_limit_state_prob + - file: analyses/ml_joplin_cge + - file: analyses/ml_slc_cge - file: analyses/multi_retrofit_optimization - file: analyses/nci_functionality - file: analyses/pipeline_dmg diff --git a/manual_jb/content/analyses.md b/manual_jb/content/analyses.md index 956b4242..73f34832 100644 --- a/manual_jb/content/analyses.md +++ b/manual_jb/content/analyses.md @@ -26,27 +26,28 @@ 24. [Interdependent Network Design Problem](analyses/indp) 25. [Joplin Computable General Equilibrium (CGE)](analyses/joplin_cge) 26. [Joplin empirical building restoration](analyses/joplin_empirical_building_restoration) -27. [Machine Learning Enabled Computable General Equilibrium (CGE) - Salt Lake City](analyses/ml_slc_cge.md) -28. [Mean damage](analyses/mean_dmg) -29. [Monte Carlo limit state probability](analyses/mc_limit_state_prob) -30. [Multi-objective retrofit optimization](analyses/multi_retrofit_optimization) -31. [Network cascading interdependency functionality](analyses/nci_functionality) -32. [Pipeline damage](analyses/pipeline_dmg) -33. [Pipeline damage with repair rate](analyses/pipeline_dmg_w_repair_rate) -34. [Pipeline functionality](analyses/pipeline_functionality) -35. [Pipeline repair cost](analyses/pipeline_repair_cost) -36. [Pipeline restoration](analyses/pipeline_restoration) -37. [Population dislocation](analyses/populationdislocation) -38. [Residential building recovery](analyses/residential_building_recovery) -39. [Road damage](analyses/road_dmg) -40. [Salt Lake City Computable General Equilibrium (CGE)](analyses/slc_cge.md) -41. [Seaside Computable General Equilibrium (CGE)](analyses/seaside_cge) -42. [Social Vulnerability](analyses/social_vulnerability) -43. [Social Vulnerability Score](analyses/social_vulnerability_score) -44. [Tornado electric power network (EPN) damage](analyses/tornadoepn_dmg) -45. [Traffic flow recovery](analyses/traffic_flow_recovery) -46. [Transportation recovery](analyses/transportation_recovery) -47. [Water facility damage](analyses/waterfacility_dmg) -48. [Water network functionality](analyses/wfn_functionality) -49. [Water facility repair cost](analyses/water_facility_repair_cost) -50. [Water facility restoration](analyses/water_facility_restoration) +27. [Machine Learning Enabled Computable General Equilibrium (CGE) - Joplin](analyses/ml_joplin_cge.md) +28. [Machine Learning Enabled Computable General Equilibrium (CGE) - Salt Lake City](analyses/ml_slc_cge.md) +29. [Mean damage](analyses/mean_dmg) +30. [Monte Carlo limit state probability](analyses/mc_limit_state_prob) +31. [Multi-objective retrofit optimization](analyses/multi_retrofit_optimization) +32. [Network cascading interdependency functionality](analyses/nci_functionality) +33. [Pipeline damage](analyses/pipeline_dmg) +34. [Pipeline damage with repair rate](analyses/pipeline_dmg_w_repair_rate) +35. [Pipeline functionality](analyses/pipeline_functionality) +36. [Pipeline repair cost](analyses/pipeline_repair_cost) +37. [Pipeline restoration](analyses/pipeline_restoration) +38. [Population dislocation](analyses/populationdislocation) +39. [Residential building recovery](analyses/residential_building_recovery) +40. [Road damage](analyses/road_dmg) +41. [Salt Lake City Computable General Equilibrium (CGE)](analyses/slc_cge.md) +42. [Seaside Computable General Equilibrium (CGE)](analyses/seaside_cge) +43. [Social Vulnerability](analyses/social_vulnerability) +44. [Social Vulnerability Score](analyses/social_vulnerability_score) +45. [Tornado electric power network (EPN) damage](analyses/tornadoepn_dmg) +46. [Traffic flow recovery](analyses/traffic_flow_recovery) +47. [Transportation recovery](analyses/transportation_recovery) +48. [Water facility damage](analyses/waterfacility_dmg) +49. [Water network functionality](analyses/wfn_functionality) +50. [Water facility repair cost](analyses/water_facility_repair_cost) +51. [Water facility restoration](analyses/water_facility_restoration) diff --git a/manual_jb/content/analyses/ml_joplin_cge.md b/manual_jb/content/analyses/ml_joplin_cge.md new file mode 100644 index 00000000..a7bf5310 --- /dev/null +++ b/manual_jb/content/analyses/ml_joplin_cge.md @@ -0,0 +1,61 @@ +# Machine Learning Enabled Computable General Equilibrium (CGE) - Joplin + +**Description** + +The "Machine Learning Enabled Computable General Equilibrium (CGE) - Joplin" analysis merges advanced machine learning with traditional CGE models to offer unprecedented insights into the economic impacts of disaster scenarios on Joplin. Trained on a comprehensive dataset of numerous simulated disasters and their economic effects, this hybrid approach excels in predicting the intricate dynamics of the city's economy under various crises. + +A computable general equilibrium (CGE) model is based on fundamental economic principles. A CGE model uses multiple data sources to reflect the interactions of households, firms, and relevant government entities as they contribute to economic activity. The model is based on (1) utility-maximizing households that supply labor and capital, using the proceeds to pay for goods and services (both locally produced and imported) and taxes; (2) the production sector, with perfectly competitive, profit-maximizing firms using intermediate inputs, capital, land, and labor +to produce goods and services for both domestic consumption and export; (3) the government sector that collects taxes and uses tax revenues in order to finance the provision of public services; and (4) the rest of the world. + + +The output of this analysis are CSV files with domestic supply, gross income, before- and post-disaster factor demand and household count. + +**Contributors** + +- Science: Charles Nicholson, Nushra Zannat, Hwayoung Jeon, Tao Lu, Harvey Cutler, Anita Pena +- Implementation: NCSA IN-CORE Dev Team + + +**Input parameters** + +key name | type | name | description +--- | --- | --- | --- +`result_name` | `str` | Output File Name prefix | Sets the file name prefix for output files. + +**Input datasets** + +key name | type | name | description +--- | --- | --- | --- +`sector_shocks` * | [`incore:capitalShocks`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:capitalShocks) | Capital shocks | Building states to capital
shocks per sector. + +**Output datasets** + +key name | type | name | description +--- | --- | --- | --- +`domestic-supply` * | [`incore:Employment`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:Employment) | Supply results | A dataset containing domestic supply results (format: CSV). +`gross-income` * | [`incore:Employment`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:Employment) | Gross income | A dataset of resulting gross income (format: CSV). +`pre-disaster-factor-demand` * | [`incore:FactorDemand`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:FactorDemand) | Factor demand | A dataset of factor demand before disaster (format: CSV). +`post-disaster-factor-demand` * | [`incore:FactorDemand`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:FactorDemand) | Factor demand | A dataset of factor demand after disaster (format: CSV). +`household-count` * | [`incore:HouseholdCount`](https://incore.ncsa.illinois.edu/semantics/api/types/incore:HouseholdCount) | Household count | A dataset of household count (format: CSV). + +(* required) + +**Execution** + +code snippet: + +``` + # Create Machine Learning Enabled CGE Joplin Model + mlcgejoplin = MlEnabledCgeJoplin(client) + + # Set analysis input datasets + mlcgejoplin.load_remote_input_dataset("sector_shocks", sector_shocks) + + # Optional parameters for file naming + mlcgejoplin.set_parameter("result_name", "test_joplin_mlcge_result") + + # Run Joplin CGE model analysis + mlcgejoplin.run_analysis() +``` + +full analysis: [ml_enabled_joplin_cge.ipynb](https://github.com/IN-CORE/incore-docs/blob/main/notebooks/ml_enabled_joplin_cge.ipynb) diff --git a/notebooks/ml_enabled_joplin_cge.ipynb b/notebooks/ml_enabled_joplin_cge.ipynb new file mode 100644 index 00000000..df6b46d8 --- /dev/null +++ b/notebooks/ml_enabled_joplin_cge.ipynb @@ -0,0 +1,740 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pyincore import IncoreClient, globals as pyglobals\n", + "from pyincore.analyses.mlenabledcgejoplin import MlEnabledCgeJoplin\n", + "\n", + "from pyincore import globals as pyglobals\n", + "\n", + "# Supress default INFO logging\n", + "\n", + "import logging\n", + "\n", + "logger = pyglobals.LOGGER\n", + "logger.setLevel(logging.CRITICAL)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connection successful to IN-CORE services. pyIncore version detected: 1.19.0\n" + ] + } + ], + "source": [ + "client = IncoreClient()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Sector shocks from the new Joplin Building einventory\n", + "sector_shocks = \"6706b903e905f3105421bfdf\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Create Machine Learning Enabled CGE Joplin Model\n", + "mlcgejoplin = MlEnabledCgeJoplin(client)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Set analysis input datasets\n", + "mlcgejoplin.load_remote_input_dataset(\"sector_shocks\", sector_shocks)\n", + "\n", + "# Optional parameters for file naming\n", + "mlcgejoplin.set_parameter(\"result_name\", \"test_joplin_mlcge_result\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Run Joplin CGE model analysis\n", + "mlcgejoplin.run_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SectorsDS0DSL
0AG_MIN2.2338512.115940
1UTIL_CONS90.29047678.769336
2MANU403.079552379.541172
3WHOLE109.810310100.189559
4RETAIL415.554335391.812303
5TRANS195.150137191.398167
\n", + "
" + ], + "text/plain": [ + " Sectors DS0 DSL\n", + "0 AG_MIN 2.233851 2.115940\n", + "1 UTIL_CONS 90.290476 78.769336\n", + "2 MANU 403.079552 379.541172\n", + "3 WHOLE 109.810310 100.189559\n", + "4 RETAIL 415.554335 391.812303\n", + "5 TRANS 195.150137 191.398167" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "domestic_supply = mlcgejoplin.get_output_dataset(\"domestic-supply\")\n", + "ds = domestic_supply.get_dataframe_from_csv()\n", + "# Display top 6 rows of output data\n", + "ds.head(6)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Household GroupY0YL
0HH112.12813312.219010
1HH255.34066953.288001
2HH3260.065807250.306049
3HH4312.703538302.477296
4HH5230.292728225.467172
\n", + "
" + ], + "text/plain": [ + " Household Group Y0 YL\n", + "0 HH1 12.128133 12.219010\n", + "1 HH2 55.340669 53.288001\n", + "2 HH3 260.065807 250.306049\n", + "3 HH4 312.703538 302.477296\n", + "4 HH5 230.292728 225.467172" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gross_income = mlcgejoplin.get_output_dataset(\"gross-income\")\n", + "gi = gross_income.get_dataframe_from_csv()\n", + "gi.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Labor GroupAG_MINUTIL_CONSMANUWHOLERETAILTRANSINFOFINANCEREALEPROF_MANADMIN_EDUHEALTHARTACCOOTHERFEDSTATECOUNTY
0L11.0234.01023.0375.01584.0883.033.0158.08.0204.02382.01782.045.02736.0596.01.06.023.0
1L21.0505.01899.0448.02644.0402.0177.0569.0253.0539.02163.04999.0125.01247.0425.015.061.0249.0
2L39.0354.02467.0814.01325.01598.0151.0147.062.01278.0615.02361.0166.0127.0127.04.018.073.0
3L41.079.0538.0-1.0615.0245.012.0-1.017.0-1.0172.0824.0-1.0-1.04.0-1.0-1.0-1.0
\n", + "
" + ], + "text/plain": [ + " Labor Group AG_MIN UTIL_CONS MANU WHOLE RETAIL TRANS INFO \\\n", + "0 L1 1.0 234.0 1023.0 375.0 1584.0 883.0 33.0 \n", + "1 L2 1.0 505.0 1899.0 448.0 2644.0 402.0 177.0 \n", + "2 L3 9.0 354.0 2467.0 814.0 1325.0 1598.0 151.0 \n", + "3 L4 1.0 79.0 538.0 -1.0 615.0 245.0 12.0 \n", + "\n", + " FINANCE REALE PROF_MAN ADMIN_EDU HEALTH ART ACCO OTHER FED \\\n", + "0 158.0 8.0 204.0 2382.0 1782.0 45.0 2736.0 596.0 1.0 \n", + "1 569.0 253.0 539.0 2163.0 4999.0 125.0 1247.0 425.0 15.0 \n", + "2 147.0 62.0 1278.0 615.0 2361.0 166.0 127.0 127.0 4.0 \n", + "3 -1.0 17.0 -1.0 172.0 824.0 -1.0 -1.0 4.0 -1.0 \n", + "\n", + " STATE COUNTY \n", + "0 6.0 23.0 \n", + "1 61.0 249.0 \n", + "2 18.0 73.0 \n", + "3 -1.0 -1.0 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pre_factor_demand = mlcgejoplin.get_output_dataset(\"pre-disaster-factor-demand\")\n", + "pre_fd = pre_factor_demand.get_dataframe_from_csv()\n", + "pre_fd.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Labor GroupAG_MINUTIL_CONSMANUWHOLERETAILTRANSINFOFINANCEREALEPROF_MANADMIN_EDUHEALTHARTACCOOTHERFEDSTATECOUNTY
0L10.963198212.8816631014.199361369.6038591551.727581875.63710231.810105148.3679997.497354201.3070382340.5665841760.28950141.7947882630.745249563.0591660.9757875.86444222.541280
1L20.961808458.2479491877.815594440.4399252583.616603397.608252170.611461533.043612239.717699530.5833392119.8739254925.343274115.6356931195.905070400.59290114.70334959.498755243.227744
2L38.772864320.4378282433.431697798.3109611291.5450141576.734337145.179908137.35657758.5863801254.955803601.2568452320.469288153.163294121.506667119.3943633.88878817.48490371.042039
3L40.95919771.516873530.857702-1.000000599.638906241.77944111.464508-1.00000015.981553-1.000000168.225150810.058297-1.000000-1.0000003.753971-1.000000-1.000000-1.000000
\n", + "
" + ], + "text/plain": [ + " Labor Group AG_MIN UTIL_CONS MANU WHOLE RETAIL \\\n", + "0 L1 0.963198 212.881663 1014.199361 369.603859 1551.727581 \n", + "1 L2 0.961808 458.247949 1877.815594 440.439925 2583.616603 \n", + "2 L3 8.772864 320.437828 2433.431697 798.310961 1291.545014 \n", + "3 L4 0.959197 71.516873 530.857702 -1.000000 599.638906 \n", + "\n", + " TRANS INFO FINANCE REALE PROF_MAN ADMIN_EDU \\\n", + "0 875.637102 31.810105 148.367999 7.497354 201.307038 2340.566584 \n", + "1 397.608252 170.611461 533.043612 239.717699 530.583339 2119.873925 \n", + "2 1576.734337 145.179908 137.356577 58.586380 1254.955803 601.256845 \n", + "3 241.779441 11.464508 -1.000000 15.981553 -1.000000 168.225150 \n", + "\n", + " HEALTH ART ACCO OTHER FED STATE \\\n", + "0 1760.289501 41.794788 2630.745249 563.059166 0.975787 5.864442 \n", + "1 4925.343274 115.635693 1195.905070 400.592901 14.703349 59.498755 \n", + "2 2320.469288 153.163294 121.506667 119.394363 3.888788 17.484903 \n", + "3 810.058297 -1.000000 -1.000000 3.753971 -1.000000 -1.000000 \n", + "\n", + " COUNTY \n", + "0 22.541280 \n", + "1 243.227744 \n", + "2 71.042039 \n", + "3 -1.000000 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "post_factor_demand = mlcgejoplin.get_output_dataset(\"post-disaster-factor-demand\")\n", + "pos_fd = post_factor_demand.get_dataframe_from_csv()\n", + "pos_fd.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Household GroupHH0HHL
0HH14126.04123.484509
1HH24977.04943.647474
2HH311487.011367.423014
3HH46847.06746.360290
4HH52827.02797.164588
\n", + "
" + ], + "text/plain": [ + " Household Group HH0 HHL\n", + "0 HH1 4126.0 4123.484509\n", + "1 HH2 4977.0 4943.647474\n", + "2 HH3 11487.0 11367.423014\n", + "3 HH4 6847.0 6746.360290\n", + "4 HH5 2827.0 2797.164588" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "household_count = mlcgejoplin.get_output_dataset(\"household-count\")\n", + "hc = household_count.get_dataframe_from_csv()\n", + "hc.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "incore", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From fd748f116d9345caed8939f9acf2aacaeed0506b Mon Sep 17 00:00:00 2001 From: Chris Navarro Date: Wed, 23 Oct 2024 21:46:50 -0500 Subject: [PATCH 6/6] Updated key in residential building recovery notebook (#431) --- notebooks/residential_building_recovery.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/residential_building_recovery.ipynb b/notebooks/residential_building_recovery.ipynb index 3e86e713..2a2c6cc5 100644 --- a/notebooks/residential_building_recovery.ipynb +++ b/notebooks/residential_building_recovery.ipynb @@ -131,7 +131,7 @@ "outputs": [], "source": [ "# Retrieve result dataset\n", - "result = res_recovery.get_output_dataset(\"residential_building_recovery\")\n", + "result = res_recovery.get_output_dataset(\"recovery\")\n", "\n", "# Convert dataset to Pandas DataFrame\n", "df = result.get_dataframe_from_csv()\n",