Merge pull request #244 from OSeMOSYS/vis-snakemake-updates

Vis snakemake updates

workflow/rules/postprocess.smk

@@ -58,20 +58,25 @@ rule visualisation:
         csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS),
         centerpoints = 'resources/data/centerpoints.csv',
         custom_nodes_centerpoints = 'resources/data/custom_nodes/centerpoints.csv',
+        color_codes = 'resources/data/color_codes.csv',
     params:
-        input_data = "results/{scenario}/data/",
+        result_input_data = "results/{scenario}/data/",
         result_data = "results/{scenario}/results/",
         scenario_figs_dir = "results/{scenario}/figures/",
-        countries = config['geographic_scope'],
+        geographic_scope = config['geographic_scope'],
         results_by_country = config['results_by_country'],
-        years = [config['endYear']],
+        start_year = config['startYear'],
+        end_year = [config['endYear']],
         custom_nodes = config['nodes_to_add'],
+        seasons = config['seasons'],
+        dayparts = config['dayparts'],
+        timeshift = config['timeshift'],
     output:
         expand('results/{{scenario}}/figures/{result_figure}.html', result_figure = RESULT_FIGURES)
     log:
         log = 'results/{scenario}/logs/visualisation.log'
-    shell: 
-        'python workflow/scripts/osemosys_global/visualise.py {params.input_data} {params.result_data} {params.scenario_figs_dir} {params.countries} {params.results_by_country} {params.years} 2> {log}'
+    script: 
+        "../scripts/osemosys_global/visualisation/visualise.py" 
 
 rule calculate_trade_flows:
     message: 

workflow/scripts/osemosys_global/constants.py

@@ -11,28 +11,4 @@
     "TECHNOLOGY":str,
     "TIMESLICE":str,
     "YEAR":int,
-}
-
-COLORS = {
-    "BIO":"darkgreen",
-    "CCG":"lightcoral",
-    "COA":"black",
-    "COG":"peru",
-    "CSP":"wheat",
-    "ELC":"gold",
-    "GAS":"orange",
-    "GEO":"darkseagreen",
-    "HYD":"dodgerblue",
-    "OCG":"firebrick",
-    "OIL":"lightgrey",
-    "OTH":"teal",
-    "PET":"grey",
-    "SOL":"gold",
-    "SPV":"gold",
-    "URN":"mediumseagreen",
-    "WAS":"darkkhaki",
-    "WAV":"navy",
-    "WOF":"violet",
-    "WON":"blueviolet",
-    "INT":"darkgreen",
 }

workflow/scripts/osemosys_global/utils.py

@@ -1,9 +1,8 @@
 """Utility Functions"""
 
 import pandas as pd
-from typing import Dict, Optional
-from pathlib import Path
-from osemosys_global.constants import SET_DTYPES
+from typing import Optional
+from constants import SET_DTYPES
 
 import logging 
 logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
@@ -22,31 +21,6 @@ def apply_timeshift(x, timeshift):
         return x + 24
     else:
         return x
-
-def read_csv(dirpath: str) -> Dict[str,pd.DataFrame]:
-    """Reads in CSVs folder
-    
-    Replace with ReadCSV.read() from otoole v1.0
-    """
-    data = {}
-    files = [Path(x) for x in Path(dirpath).iterdir()]
-    for f in files:
-        data[f.stem] = pd.read_csv(f)
-    return data
-
-def filter_transmission_techs(df: pd.DataFrame, column_name: str = "TECHNOLOGY") -> pd.DataFrame:
-    """Filters out only transmission technologies
-    
-    Arguments: 
-        df: pd.DataFrame
-            otoole formatted dataframe 
-        column_name: str
-            Column name to filter on 
-            
-    Returns: 
-        pd.DataFrame
-    """
-    return df.loc[df[column_name].str.startswith("TRN")].reset_index(drop=True)
 
 def apply_dtypes(df:pd.DataFrame, name: Optional[str]) -> pd.DataFrame:
     """Sets datatypes on dataframe"""

workflow/scripts/osemosys_global/visualisation/constants.py

@@ -28,4 +28,41 @@
     'Oct': 31,
     'Nov': 30,
     'Dec': 31,
+}
+
+COLORS = {
+    "BIO":"darkgreen",
+    "CCG":"lightcoral",
+    "COA":"black",
+    "COG":"peru",
+    "CSP":"wheat",
+    "ELC":"gold",
+    "GAS":"orange",
+    "GEO":"darkseagreen",
+    "HYD":"dodgerblue",
+    "OCG":"firebrick",
+    "OIL":"lightgrey",
+    "OTH":"teal",
+    "PET":"grey",
+    "SOL":"gold",
+    "SPV":"gold",
+    "URN":"mediumseagreen",
+    "WAS":"darkkhaki",
+    "WAV":"navy",
+    "WOF":"violet",
+    "WON":"blueviolet",
+    "INT":"darkgreen",
+}
+
+SET_DTYPES = {
+    "DAILYTIMEBRACKET": int,
+    "EMISSION":str,
+    "FUEL":str,
+    "MODE_OF_OPERATION":int,
+    "REGION":str,
+    "SEASON":str,
+    "STORAGE":str,
+    "TECHNOLOGY":str,
+    "TIMESLICE":str,
+    "YEAR":int,
 }

workflow/scripts/osemosys_global/visualisation/data.py

@@ -1,6 +1,6 @@
 """Getter functions for data to plot"""
 
-from .utils import powerplant_filter, transform_ts
+from utils import powerplant_filter, transform_ts
 import pandas as pd
 from typing import Dict
 import logging 
@@ -48,7 +48,14 @@ def get_generation_annual_data(data: Dict[str,pd.DataFrame], country:str =None):
                     as_index=False)['VALUE'].sum()
     return df
 
-def get_generation_ts_data(input_data: Dict[str,pd.DataFrame], result_data: Dict[str,pd.DataFrame], country:str =None):
+def get_generation_ts_data(
+        seasons, 
+        dayparts,
+        timeshift,
+        start_year,
+        end_year,
+        input_data: Dict[str,pd.DataFrame], 
+        result_data: Dict[str,pd.DataFrame], country:str =None):
     """Gets data for plotting generation by time slice
     
     Arguments: 
@@ -67,5 +74,11 @@ def get_generation_ts_data(input_data: Dict[str,pd.DataFrame], result_data: Dict
     df = result_data["ProductionByTechnology"]
     df = powerplant_filter(df, country)
     df.VALUE = df.VALUE.astype('float64')
-    df = transform_ts(input_data, df)
+    df = transform_ts(seasons, 
+                      dayparts,
+                      timeshift,
+                      start_year,
+                      end_year,
+                      input_data, 
+                      df)
     return df

workflow/scripts/osemosys_global/visualisation/utils.py

@@ -1,12 +1,10 @@
 """Module for utility plotting functions."""
 
 import pandas as pd
-import os 
-from osemosys_global.configuration import ConfigFile, ConfigPaths
 from typing import Dict, List, Union, Tuple
 import itertools
 from pathlib import Path
-from osemosys_global.visualisation.constants import DAYS_PER_MONTH, MONTH_NAMES
+from constants import DAYS_PER_MONTH, MONTH_NAMES
 from osemosys_global.utils import apply_timeshift
 import cartopy.crs as ccrs
 import cartopy.feature as cfeature
@@ -15,26 +13,45 @@
 import logging 
 logger = logging.getLogger(__name__)
 
-def get_color_codes() -> Dict:
+def get_years(start: int, end: int) -> range:
+    return range(start, end + 1)
+
+def read_csv(dirpath: str) -> Dict[str,pd.DataFrame]:
+    """Reads in CSVs folder
+    
+    Replace with ReadCSV.read() from otoole v1.0
+    """
+    data = {}
+    files = [Path(x) for x in Path(dirpath).iterdir()]
+    for f in files:
+        data[f.stem] = pd.read_csv(f)
+    return data
+
+def filter_transmission_techs(df: pd.DataFrame, column_name: str = "TECHNOLOGY") -> pd.DataFrame:
+    """Filters out only transmission technologies
+    
+    Arguments: 
+        df: pd.DataFrame
+            otoole formatted dataframe 
+        column_name: str
+            Column name to filter on 
+            
+    Returns: 
+        pd.DataFrame
+    """
+    return df.loc[df[column_name].str.startswith("TRN")].reset_index(drop=True)
+
+def get_color_codes(color_codes) -> Dict:
     """Get color naming dictionary.
     
     Return:
         Dictionary with tech and color name map
     """
-    try:
-        config_paths = ConfigPaths()
-        input_data_dir = config_paths.input_data_dir
-    except FileNotFoundError:
-        input_data_dir = str(Path("resources", "data"))
-    name_colour_codes = pd.read_csv(os.path.join(input_data_dir,
-                                                'color_codes.csv'
-                                                ),
-                                   encoding='latin-1')
 
     # Get colour mapping dictionary
     color_dict = dict([(n, c) for n, c
-                   in zip(name_colour_codes.tech_id,
-                          name_colour_codes.colour)])
+                   in zip(color_codes.tech_id,
+                          color_codes.colour)])
     return color_dict
 
 def powerplant_filter(df: pd.DataFrame, country:str = None) -> pd.DataFrame:
@@ -64,7 +81,13 @@ def powerplant_filter(df: pd.DataFrame, country:str = None) -> pd.DataFrame:
             inplace=True)
     return filtered_df
 
-def transform_ts(data:Dict[str, pd.DataFrame], df:pd.DataFrame) -> pd.DataFrame:
+def transform_ts(seasons, 
+                 dayparts,
+                 timeshift,
+                 start_year,
+                 end_year,
+                 data:Dict[str, pd.DataFrame], 
+                 df:pd.DataFrame) -> pd.DataFrame:
     """Adds month, hour, year columns to timesliced data. 
     
     Arguments:
@@ -79,25 +102,21 @@ def transform_ts(data:Dict[str, pd.DataFrame], df:pd.DataFrame) -> pd.DataFrame:
 
     generation = list(data["TECHNOLOGY"]["VALUE"].unique())
 
-    config = ConfigFile('config')
-    if not config.file_path.exists():
-        config.file_path = "config/config.yaml"
-    seasons_raw = config.get('seasons')
     seasonsData = []
 
-    for s, months in seasons_raw.items():
+    for s, months in seasons.items():
         for month in months:
             seasonsData.append([month, s]) 
     seasons_df = pd.DataFrame(seasonsData, 
                               columns=['month', 'season'])
     seasons_df = seasons_df.sort_values(by=['month']).reset_index(drop=True)
-    dayparts_raw = config.get('dayparts')
+
     daypartData = []
-    for dp, hr in dayparts_raw.items():
+    for dp, hr in dayparts.items():
         daypartData.append([dp, hr[0], hr[1]])
     dayparts_df = pd.DataFrame(daypartData,
                                columns=['daypart', 'start_hour', 'end_hour'])
-    timeshift = config.get('timeshift')
+
     dayparts_df['start_hour'] = dayparts_df['start_hour'].map(lambda x: apply_timeshift(x, timeshift))
     dayparts_df['end_hour'] = dayparts_df['end_hour'].map(lambda x: apply_timeshift(x, timeshift))
 
@@ -114,7 +133,7 @@ def transform_ts(data:Dict[str, pd.DataFrame], df:pd.DataFrame) -> pd.DataFrame:
                      )
     seasons_df['days'] = seasons_df['season'].map(days_dict)
 
-    years = config.get_years()
+    years = get_years(start_year, end_year[0])
 
     seasons_dict = dict(zip(list(seasons_df['month']),
                             list(seasons_df['season'])
@@ -210,7 +229,7 @@ def get_map(extent:List[Union[int,float]] = None) -> Tuple[plt.figure, plt.axes]
     fig, ax = plt.subplots(subplot_kw={"projection": mrc})
     ax.add_feature(cfeature.NaturalEarthFeature('physical', 'land', '50m', edgecolor='face', facecolor='lightgrey'))
     ax.add_feature(cfeature.NaturalEarthFeature('physical', 'ocean', '50m', edgecolor='black', linewidth = 0.3, facecolor='#46bcec'))
-    ax.add_feature(cfeature.BORDERS, color="black", linewidth = 0.3)
+    ax.add_feature(cfeature.BORDERS, edgecolor="black", linewidth = 0.3)
 
     if extent:
         ax.set_extent(extent)