diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 6393adf..d288785 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -58,20 +58,25 @@ rule visualisation: csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), centerpoints = 'resources/data/centerpoints.csv', custom_nodes_centerpoints = 'resources/data/custom_nodes/centerpoints.csv', + color_codes = 'resources/data/color_codes.csv', params: - input_data = "results/{scenario}/data/", + result_input_data = "results/{scenario}/data/", result_data = "results/{scenario}/results/", scenario_figs_dir = "results/{scenario}/figures/", - countries = config['geographic_scope'], + geographic_scope = config['geographic_scope'], results_by_country = config['results_by_country'], - years = [config['endYear']], + start_year = config['startYear'], + end_year = [config['endYear']], custom_nodes = config['nodes_to_add'], + seasons = config['seasons'], + dayparts = config['dayparts'], + timeshift = config['timeshift'], output: expand('results/{{scenario}}/figures/{result_figure}.html', result_figure = RESULT_FIGURES) log: log = 'results/{scenario}/logs/visualisation.log' - shell: - 'python workflow/scripts/osemosys_global/visualise.py {params.input_data} {params.result_data} {params.scenario_figs_dir} {params.countries} {params.results_by_country} {params.years} 2> {log}' + script: + "../scripts/osemosys_global/visualisation/visualise.py" rule calculate_trade_flows: message: diff --git a/workflow/scripts/osemosys_global/constants.py b/workflow/scripts/osemosys_global/constants.py index ab1df9a..f7212d0 100644 --- a/workflow/scripts/osemosys_global/constants.py +++ b/workflow/scripts/osemosys_global/constants.py @@ -11,28 +11,4 @@ "TECHNOLOGY":str, "TIMESLICE":str, "YEAR":int, -} - -COLORS = { - "BIO":"darkgreen", - "CCG":"lightcoral", - "COA":"black", - "COG":"peru", - "CSP":"wheat", - "ELC":"gold", - "GAS":"orange", - "GEO":"darkseagreen", - "HYD":"dodgerblue", - "OCG":"firebrick", - "OIL":"lightgrey", - "OTH":"teal", - "PET":"grey", - "SOL":"gold", - "SPV":"gold", - "URN":"mediumseagreen", - "WAS":"darkkhaki", - "WAV":"navy", - "WOF":"violet", - "WON":"blueviolet", - "INT":"darkgreen", } \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/utils.py b/workflow/scripts/osemosys_global/utils.py index 9b5a9a0..185bf47 100644 --- a/workflow/scripts/osemosys_global/utils.py +++ b/workflow/scripts/osemosys_global/utils.py @@ -1,9 +1,8 @@ """Utility Functions""" import pandas as pd -from typing import Dict, Optional -from pathlib import Path -from osemosys_global.constants import SET_DTYPES +from typing import Optional +from constants import SET_DTYPES import logging logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) @@ -22,31 +21,6 @@ def apply_timeshift(x, timeshift): return x + 24 else: return x - -def read_csv(dirpath: str) -> Dict[str,pd.DataFrame]: - """Reads in CSVs folder - - Replace with ReadCSV.read() from otoole v1.0 - """ - data = {} - files = [Path(x) for x in Path(dirpath).iterdir()] - for f in files: - data[f.stem] = pd.read_csv(f) - return data - -def filter_transmission_techs(df: pd.DataFrame, column_name: str = "TECHNOLOGY") -> pd.DataFrame: - """Filters out only transmission technologies - - Arguments: - df: pd.DataFrame - otoole formatted dataframe - column_name: str - Column name to filter on - - Returns: - pd.DataFrame - """ - return df.loc[df[column_name].str.startswith("TRN")].reset_index(drop=True) def apply_dtypes(df:pd.DataFrame, name: Optional[str]) -> pd.DataFrame: """Sets datatypes on dataframe""" diff --git a/workflow/scripts/osemosys_global/visualisation/constants.py b/workflow/scripts/osemosys_global/visualisation/constants.py index d8c3860..05700f4 100644 --- a/workflow/scripts/osemosys_global/visualisation/constants.py +++ b/workflow/scripts/osemosys_global/visualisation/constants.py @@ -28,4 +28,41 @@ 'Oct': 31, 'Nov': 30, 'Dec': 31, +} + +COLORS = { + "BIO":"darkgreen", + "CCG":"lightcoral", + "COA":"black", + "COG":"peru", + "CSP":"wheat", + "ELC":"gold", + "GAS":"orange", + "GEO":"darkseagreen", + "HYD":"dodgerblue", + "OCG":"firebrick", + "OIL":"lightgrey", + "OTH":"teal", + "PET":"grey", + "SOL":"gold", + "SPV":"gold", + "URN":"mediumseagreen", + "WAS":"darkkhaki", + "WAV":"navy", + "WOF":"violet", + "WON":"blueviolet", + "INT":"darkgreen", +} + +SET_DTYPES = { + "DAILYTIMEBRACKET": int, + "EMISSION":str, + "FUEL":str, + "MODE_OF_OPERATION":int, + "REGION":str, + "SEASON":str, + "STORAGE":str, + "TECHNOLOGY":str, + "TIMESLICE":str, + "YEAR":int, } \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/visualisation/data.py b/workflow/scripts/osemosys_global/visualisation/data.py index d632500..fb99ffe 100644 --- a/workflow/scripts/osemosys_global/visualisation/data.py +++ b/workflow/scripts/osemosys_global/visualisation/data.py @@ -1,6 +1,6 @@ """Getter functions for data to plot""" -from .utils import powerplant_filter, transform_ts +from utils import powerplant_filter, transform_ts import pandas as pd from typing import Dict import logging @@ -48,7 +48,14 @@ def get_generation_annual_data(data: Dict[str,pd.DataFrame], country:str =None): as_index=False)['VALUE'].sum() return df -def get_generation_ts_data(input_data: Dict[str,pd.DataFrame], result_data: Dict[str,pd.DataFrame], country:str =None): +def get_generation_ts_data( + seasons, + dayparts, + timeshift, + start_year, + end_year, + input_data: Dict[str,pd.DataFrame], + result_data: Dict[str,pd.DataFrame], country:str =None): """Gets data for plotting generation by time slice Arguments: @@ -67,5 +74,11 @@ def get_generation_ts_data(input_data: Dict[str,pd.DataFrame], result_data: Dict df = result_data["ProductionByTechnology"] df = powerplant_filter(df, country) df.VALUE = df.VALUE.astype('float64') - df = transform_ts(input_data, df) + df = transform_ts(seasons, + dayparts, + timeshift, + start_year, + end_year, + input_data, + df) return df \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/visualisation/utils.py b/workflow/scripts/osemosys_global/visualisation/utils.py index 79fbd1e..a2319c7 100644 --- a/workflow/scripts/osemosys_global/visualisation/utils.py +++ b/workflow/scripts/osemosys_global/visualisation/utils.py @@ -1,12 +1,10 @@ """Module for utility plotting functions.""" import pandas as pd -import os -from osemosys_global.configuration import ConfigFile, ConfigPaths from typing import Dict, List, Union, Tuple import itertools from pathlib import Path -from osemosys_global.visualisation.constants import DAYS_PER_MONTH, MONTH_NAMES +from constants import DAYS_PER_MONTH, MONTH_NAMES from osemosys_global.utils import apply_timeshift import cartopy.crs as ccrs import cartopy.feature as cfeature @@ -15,26 +13,45 @@ import logging logger = logging.getLogger(__name__) -def get_color_codes() -> Dict: +def get_years(start: int, end: int) -> range: + return range(start, end + 1) + +def read_csv(dirpath: str) -> Dict[str,pd.DataFrame]: + """Reads in CSVs folder + + Replace with ReadCSV.read() from otoole v1.0 + """ + data = {} + files = [Path(x) for x in Path(dirpath).iterdir()] + for f in files: + data[f.stem] = pd.read_csv(f) + return data + +def filter_transmission_techs(df: pd.DataFrame, column_name: str = "TECHNOLOGY") -> pd.DataFrame: + """Filters out only transmission technologies + + Arguments: + df: pd.DataFrame + otoole formatted dataframe + column_name: str + Column name to filter on + + Returns: + pd.DataFrame + """ + return df.loc[df[column_name].str.startswith("TRN")].reset_index(drop=True) + +def get_color_codes(color_codes) -> Dict: """Get color naming dictionary. Return: Dictionary with tech and color name map """ - try: - config_paths = ConfigPaths() - input_data_dir = config_paths.input_data_dir - except FileNotFoundError: - input_data_dir = str(Path("resources", "data")) - name_colour_codes = pd.read_csv(os.path.join(input_data_dir, - 'color_codes.csv' - ), - encoding='latin-1') # Get colour mapping dictionary color_dict = dict([(n, c) for n, c - in zip(name_colour_codes.tech_id, - name_colour_codes.colour)]) + in zip(color_codes.tech_id, + color_codes.colour)]) return color_dict def powerplant_filter(df: pd.DataFrame, country:str = None) -> pd.DataFrame: @@ -64,7 +81,13 @@ def powerplant_filter(df: pd.DataFrame, country:str = None) -> pd.DataFrame: inplace=True) return filtered_df -def transform_ts(data:Dict[str, pd.DataFrame], df:pd.DataFrame) -> pd.DataFrame: +def transform_ts(seasons, + dayparts, + timeshift, + start_year, + end_year, + data:Dict[str, pd.DataFrame], + df:pd.DataFrame) -> pd.DataFrame: """Adds month, hour, year columns to timesliced data. Arguments: @@ -79,25 +102,21 @@ def transform_ts(data:Dict[str, pd.DataFrame], df:pd.DataFrame) -> pd.DataFrame: generation = list(data["TECHNOLOGY"]["VALUE"].unique()) - config = ConfigFile('config') - if not config.file_path.exists(): - config.file_path = "config/config.yaml" - seasons_raw = config.get('seasons') seasonsData = [] - for s, months in seasons_raw.items(): + for s, months in seasons.items(): for month in months: seasonsData.append([month, s]) seasons_df = pd.DataFrame(seasonsData, columns=['month', 'season']) seasons_df = seasons_df.sort_values(by=['month']).reset_index(drop=True) - dayparts_raw = config.get('dayparts') + daypartData = [] - for dp, hr in dayparts_raw.items(): + for dp, hr in dayparts.items(): daypartData.append([dp, hr[0], hr[1]]) dayparts_df = pd.DataFrame(daypartData, columns=['daypart', 'start_hour', 'end_hour']) - timeshift = config.get('timeshift') + dayparts_df['start_hour'] = dayparts_df['start_hour'].map(lambda x: apply_timeshift(x, timeshift)) dayparts_df['end_hour'] = dayparts_df['end_hour'].map(lambda x: apply_timeshift(x, timeshift)) @@ -114,7 +133,7 @@ def transform_ts(data:Dict[str, pd.DataFrame], df:pd.DataFrame) -> pd.DataFrame: ) seasons_df['days'] = seasons_df['season'].map(days_dict) - years = config.get_years() + years = get_years(start_year, end_year[0]) seasons_dict = dict(zip(list(seasons_df['month']), list(seasons_df['season']) @@ -210,7 +229,7 @@ def get_map(extent:List[Union[int,float]] = None) -> Tuple[plt.figure, plt.axes] fig, ax = plt.subplots(subplot_kw={"projection": mrc}) ax.add_feature(cfeature.NaturalEarthFeature('physical', 'land', '50m', edgecolor='face', facecolor='lightgrey')) ax.add_feature(cfeature.NaturalEarthFeature('physical', 'ocean', '50m', edgecolor='black', linewidth = 0.3, facecolor='#46bcec')) - ax.add_feature(cfeature.BORDERS, color="black", linewidth = 0.3) + ax.add_feature(cfeature.BORDERS, edgecolor="black", linewidth = 0.3) if extent: ax.set_extent(extent) diff --git a/workflow/scripts/osemosys_global/visualise.py b/workflow/scripts/osemosys_global/visualisation/visualise.py similarity index 74% rename from workflow/scripts/osemosys_global/visualise.py rename to workflow/scripts/osemosys_global/visualisation/visualise.py index 0a115e4..a471336 100644 --- a/workflow/scripts/osemosys_global/visualise.py +++ b/workflow/scripts/osemosys_global/visualisation/visualise.py @@ -2,64 +2,77 @@ pd.set_option('mode.chained_assignment', None) import plotly.express as px import os -import sys from typing import List from sklearn.preprocessing import MinMaxScaler from typing import Dict -from osemosys_global.utils import read_csv, filter_transmission_techs -from osemosys_global.visualisation.utils import(get_color_codes, get_map, plot_map_trn_line, plot_map_text) -from osemosys_global.visualisation.data import(get_total_capacity_data, get_generation_annual_data, - get_generation_ts_data) -import osemosys_global.constants as constants -from configuration import ConfigFile, ConfigPaths +from utils import(read_csv, + filter_transmission_techs, + get_color_codes, + get_map, + plot_map_trn_line, + plot_map_text) + +from data import( + get_total_capacity_data, + get_generation_annual_data, + get_generation_ts_data) + +from constants import COLORS def main( - input_data: pd.DataFrame, + result_input_data: pd.DataFrame, result_data: pd.DataFrame, - centerpoints: pd.DataFrame, + centerpoints: pd.DataFrame, + color_codes: pd.DataFrame, + seasons: dict, + dayparts: dict, + timeshift: int, + start_year: int, custom_nodes: List[str], custom_nodes_centerpoints : pd.DataFrame, scenario_figs_dir: str, - countries: List[str], + geographic_scope: List[str], results_by_country: bool = True, - years: List[int] = [2050], + end_year: List[int] = [2050], ): - """Creates system level and country level graphs.""" - + """Creates system level and country level graphs.""" + # Check for output directory try: os.makedirs(scenario_figs_dir) except FileExistsError: pass - + # Get system level results plot_total_capacity(result_data, scenario_figs_dir, country=None) - plot_generation_annual(result_data, scenario_figs_dir, country=None) - plot_generation_hourly(input_data, result_data, scenario_figs_dir, country=None) + plot_generation_annual(color_codes, result_data, scenario_figs_dir, country=None) + plot_generation_hourly(seasons, dayparts, timeshift, start_year, end_year, + color_codes, result_input_data, result_data, scenario_figs_dir, + country=None) # If producing by country results, check for folder structure if results_by_country: - for country in countries: + for country in geographic_scope: try: os.makedirs(os.path.join(scenario_figs_dir, country)) except FileExistsError: pass plot_total_capacity(result_data, scenario_figs_dir, country=country) - plot_generation_annual(result_data, scenario_figs_dir, country=country) + plot_generation_annual(color_codes, result_data, scenario_figs_dir, country=country) # Creates transmission maps by year - for year in years: - plot_transmission_capacity(custom_nodes, centerpoints, custom_nodes_centerpoints, - result_data, scenario_figs_dir, year) - - plot_transmission_flow(custom_nodes, centerpoints, custom_nodes_centerpoints, - result_data, scenario_figs_dir, year) + + plot_transmission_capacity(custom_nodes, centerpoints, custom_nodes_centerpoints, + result_data, scenario_figs_dir, end_year) + + plot_transmission_flow(custom_nodes, centerpoints, custom_nodes_centerpoints, + result_data, scenario_figs_dir, end_year) def plot_total_capacity(data: Dict[str,pd.DataFrame], save_dir: str, country:str = None) -> None: """Plots total capacity chart - + Arguments: data: Dict[str,pd.DataFrame] Result data @@ -71,8 +84,8 @@ def plot_total_capacity(data: Dict[str,pd.DataFrame], save_dir: str, country:str """ df = get_total_capacity_data(data, country=country) - plot_colors = constants.COLORS - + plot_colors = COLORS + if not country: # System level titles graph_title = 'Total System Capacity' legend_title = 'Powerplant' @@ -106,7 +119,8 @@ def plot_total_capacity(data: Dict[str,pd.DataFrame], save_dir: str, country:str return fig.write_html(fig_file) -def plot_generation_annual(data: Dict[str,pd.DataFrame], save_dir: str, country:str = None) -> None: +def plot_generation_annual(color_codes, data: Dict[str,pd.DataFrame], + save_dir: str, country:str = None) -> None: """Plots total annual generation Arguments: @@ -118,7 +132,7 @@ def plot_generation_annual(data: Dict[str,pd.DataFrame], save_dir: str, country: """ df = get_generation_annual_data(data, country=country) - plot_colors = get_color_codes() + plot_colors = get_color_codes(color_codes) if not country: # System level titles graph_title = 'Total System Generation' @@ -155,11 +169,18 @@ def plot_generation_annual(data: Dict[str,pd.DataFrame], save_dir: str, country: def plot_generation_hourly( - input_data: Dict[str,pd.DataFrame], + seasons, + dayparts, + timeshift, + start_year, + end_year, + color_codes, + result_input_data: Dict[str,pd.DataFrame], result_data: Dict[str,pd.DataFrame], save_dir: str, country:str = None ) -> None: + """Plots total annual generation Arguments: @@ -170,8 +191,16 @@ def plot_generation_hourly( system level """ - df = get_generation_ts_data(input_data, result_data, country=country) - plot_colors = get_color_codes() + df = get_generation_ts_data(seasons, + dayparts, + timeshift, + start_year, + end_year, + result_input_data, + result_data, + country=country) + + plot_colors = get_color_codes(color_codes) fig = px.area(df, x='HOUR', @@ -211,7 +240,7 @@ def plot_transmission_capacity( save_dir: str, year:int ) -> None: - + # get result data total_cap_annual = result_data["TotalCapacityAnnual"] trn = filter_transmission_techs(total_cap_annual) @@ -249,8 +278,9 @@ def plot_transmission_capacity( # assign line widths based on result data scaler = MinMaxScaler() maxlinewidth = 3 - trn['line_width'] = (scaler.fit_transform(trn[['VALUE']]) * maxlinewidth).round(1) - trn['line_width'].where(trn['line_width'] >= 0.3, 0.3, inplace = True) + trn['line_width'] = (scaler.fit_transform(trn[['VALUE']]) * maxlinewidth).round(1) + trn.loc[trn['line_width'] <= 0.3, 'line_width'] = 0.3 + # get unique nodes to plot nodes_to_plot = {} @@ -272,7 +302,7 @@ def plot_transmission_capacity( fig, ax = get_map(extent=extent) # generates all lines and map text - df_year = trn.loc[trn['YEAR'] == int(year)] + df_year = trn.loc[trn['YEAR'] == int(year[0])] for y in df_year.index.unique(): # get data to plot @@ -355,7 +385,7 @@ def plot_transmission_flow( scaler = MinMaxScaler() maxlinewidth = 3 prd['line_width'] = (scaler.fit_transform(prd[['VALUE']]) * maxlinewidth).round(1) - prd['line_width'].where(prd['line_width'] >= 0.3, 0.3, inplace = True) + prd.loc[prd['line_width'] <= 0.3, 'line_width'] = 0.3 # get unique nodes to plot nodes_to_plot = {} @@ -376,7 +406,7 @@ def plot_transmission_flow( fig, ax = get_map(extent=extent) # generates all lines and map text - df_year = prd.loc[prd['YEAR'] == int(year)] + df_year = prd.loc[prd['YEAR'] == int(year[0])] for y in df_year.index.unique(): # get data to plot @@ -420,29 +450,56 @@ def plot_transmission_flow( ) if __name__ == '__main__': - try: - config_paths = ConfigPaths() - config = ConfigFile('config') - scenario_figs_dir = config_paths.scenario_figs_dir - results_by_country = config.get('results_by_country') - countries = config.get('geographic_scope') - input_data = read_csv(config_paths.scenario_data_dir) - result_data = read_csv(config_paths.scenario_results_dir) - years = [config.get('endYear')] - centerpoints = pd.read_csv(os.path.join(config_paths.input_data_dir, "centerpoints.csv")) - custom_nodes = config.get('nodes_to_add') - custom_nodes_centerpoints = pd.read_csv(os.path.join(config_paths.custom_nodes_dir, "centerpoints.csv")) - - main(input_data, - result_data, - centerpoints, - custom_nodes, - custom_nodes_centerpoints, - scenario_figs_dir, - countries, - results_by_country, - years) - - except FileNotFoundError: - print(f"Usage: python {sys.argv[0]} ") - sys.exit(1) + + if "snakemake" in globals(): + + scenario_figs_dir = snakemake.params.scenario_figs_dir + results_by_country = snakemake.params.results_by_country + geographic_scope = snakemake.params.geographic_scope + result_input_data = read_csv(snakemake.params.result_input_data) + result_data = read_csv(snakemake.params.result_data) + start_year = snakemake.params.start_year + end_year = snakemake.params.end_year + centerpoints = pd.read_csv(snakemake.input.centerpoints) + custom_nodes = snakemake.params.custom_nodes + custom_nodes_centerpoints = pd.read_csv(snakemake.input.custom_nodes_centerpoints) + color_codes = pd.read_csv(snakemake.input.color_codes) + seasons = snakemake.params.seasons + dayparts = snakemake.params.dayparts + timeshift = snakemake.params.timeshift + + else: + + scenario_figs_dir = "results/test/figures/" + results_by_country = True + geographic_scope = ["BTN", "IND"] + result_input_data = read_csv("results/test/data/") + result_data = read_csv("results/test/results/") + start_year = 2021 + end_year = 2050 + centerpoints = pd.read_csv("resources/data/centerpoints.csv") + custom_nodes = [] + custom_nodes_centerpoints = pd.read_csv("resources/data/custom_nodes/centerpoints.csv") + color_codes = pd.read_csv("resources/data/color_codes.csv") + seasons = {'S1': [1, 2, 3, 4, 5, 6], + 'S2': [7, 8, 9, 10, 11, 12]} + dayparts = {'D1': [1, 7], + 'D2': [7, 13], + 'D3': [13, 19], + 'D4': [19, 25]} + timeshift = 0 + + main(result_input_data, + result_data, + centerpoints, + color_codes, + seasons, + dayparts, + timeshift, + start_year, + custom_nodes, + custom_nodes_centerpoints, + scenario_figs_dir, + geographic_scope, + results_by_country, + end_year) \ No newline at end of file