diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..42698d6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM continuumio/miniconda3:4.10.3 + +RUN mkdir src + +WORKDIR src + +COPY environment.yml . +RUN conda env update -f environment.yml -n base + +COPY data /data + +COPY script.py . + +CMD [ "python", "script.py"] + diff --git a/README.md b/README.md new file mode 100644 index 0000000..1ca9510 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# Friction-coeffs +This model takes friction coefficient polygons supplied by the user, clips the data to the domain and ensures the data is in the correct projection. + +## Description +The CityCAT model can use friction coefficient polygons to set the permeability in specific locations. This model accepts data in .gpkg format, clips the data to the selected area, and ensures all data is in the same projection. If the file sizes are too large, multiple .gpkgs can be added directly, or zipped. + +The new friction coefficient is set as "Value" in the gpkg file + +## Input Parameters + + +## Input Files (data slots) +* Boundary + * Description: A .gpkg of the geographical area of interest. + * Location: /data/inputs/boundary +* Parameters + * Description: location and projection + * Location: /data/inputs/parameters +* Friction-ceffs + * Description: A .gpkg with the new friction coeffs set as "Value" + * Location: /data/inputs/friction_coeffs + + +## Outputs +* The model should output only one file - a .gpkg file of the chosen area containing the friction coeffs of interest with the friction coefficientsset as the "Value" + * Location: /data/outputs/friction_coeffs diff --git a/data/inputs/boundary/longbenton.gpkg b/data/inputs/boundary/longbenton.gpkg new file mode 100644 index 0000000..050cfa0 Binary files /dev/null and b/data/inputs/boundary/longbenton.gpkg differ diff --git a/data/inputs/friction_coeffs/friction-coeffs-longbenton.gpkg b/data/inputs/friction_coeffs/friction-coeffs-longbenton.gpkg new file mode 100644 index 0000000..8d3820e Binary files /dev/null and b/data/inputs/friction_coeffs/friction-coeffs-longbenton.gpkg differ diff --git a/data/inputs/parameters/parameters_England-Longbenton-parameters.csv b/data/inputs/parameters/parameters_England-Longbenton-parameters.csv new file mode 100644 index 0000000..515d071 --- /dev/null +++ b/data/inputs/parameters/parameters_England-Longbenton-parameters.csv @@ -0,0 +1,4 @@ +PARAMETER,VALUE +COUNTRY,England +LOCATION,Longbenton +PROJECTION,27700 diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..726d3d3 --- /dev/null +++ b/environment.yml @@ -0,0 +1,11 @@ +dependencies: + - python=3.9 + - rasterio=1.2.10 + - geopandas=0.12.2 + - xarray=2023.6.0 + - scipy=1.13.0 + - pip + - pip: + - netcdf4==1.6.5 + - rioxarray==0.15.0 + - matplotlib-scalebar==0.7.2 \ No newline at end of file diff --git a/model-definition.yml b/model-definition.yml new file mode 100644 index 0000000..3ae8f54 --- /dev/null +++ b/model-definition.yml @@ -0,0 +1,51 @@ +kind: M +api_version: v1beta3 +metadata: + display_name: Global Urban_Flooding:Friction Coeffs + name: citycat-friction-coeffs + publisher: Newcastle University + summary: CityCAT is a tool for modelling, analysis and visualisation of surface water flooding. + source_code: https://github.com/OpenCLIM/citycat-dafni + description: > + CityCAT (City Catchment Analysis Tool) is a unique software tool for modelling, analysis and visualisation of surface water flooding. + CityCAT enables rapid assessment of combined pluvial and fluvial flood risk and allows assessment of the effects of different flood alleviation measures. + This DAFNI model generates input data for CityCAT, runs a simulation and then converts the output data. + All input data is assumed to be projected in [OSGB 1936](https://epsg.io/27700). + The domain is generated either using a boundary polygon or a combination of centroid location and size. + A rainfall total can either be specified directly or extracted from FUTURE-DRAINAGE based on a return period, duration and time horizon. + The storm profile is generated using the FEH summer profile. + The effects of buildings, green areas and inflow boundary conditions can be included. + Results are provided in a range of formats and a metadata JSON file is created which can be used to create DAFNI datasets. + contact_point_name: steve birkinshaw + contact_point_email: s.j.birkinshaw@ncl.ac.uk + +spec: + inputs: + dataslots: + - name: Boundary File + description: + A .gpkg or .shp file containing the boundary of the location of interest. + path: inputs/boundary/ + required: false + + - name: Friction Coeffs file + description: + This file should be shp format and the location of the non-default friction coeffs and their value. + default: + - - d7ed46bc-7f9b-4f5a-b0c4-b36f5307821b + path: inputs/friction_coeffs + required: true + + - name: Parameters + description: + All input parameters and their values are stored in a csv file. + path: inputs/parameters/ + required: false + + + outputs: + datasets: + - name: outputs/friction_coeffs/* + type: folder + description: + A gpgk of friction coeff data for the city of interest. diff --git a/readme.txt b/readme.txt new file mode 100644 index 0000000..1501d2b --- /dev/null +++ b/readme.txt @@ -0,0 +1,208 @@ +cmd in folder C:\Users\steve\Documents\CityCAT-dafni\global-flood-impacts-frictioncoeffs +start docker desktop +docker build -t friction-coeffs . +Run container in Docker Desktop. In Images find friction-coeffs. Click on the play button. Then "optional settings" and call "Container name" something like "global-flood-impact-friction-coeffs" +or docker run --name global-flood-impact-friction-coeffs friction-coeffs + +docker cp global-flood-impact-friction-coeffs:/data/outputs/friction_coeffs/longbenton.gpkg ./longbenton.gpkg + +docker save -o friction-coeffs.tar friction-coeffs +compress friction-coeffs.tar to friction-coeffs.tar.gz using 7Zip add to archive + + + + + + + + + +Superceded stuff +**************** + + +Anaconda3 prompt +conda activate myenv2 +cd C:\Users\steve\Documents\CityCAT-dafni\global-flood-impacts-frictioncoeffs-0.0.1 +setenv.bat +spyder + +in two places: +filename=file_path[0].split("\\") +#filename=file_path[0].split("/") + + +conda create -n myenv rasterio geopandasconda activate myenv +pip install citycatio +pip install pyogrio +pip install spyder + + +myenv citycatio files are in C:\Users\steve\anaconda3\envs\myenv\Lib\site-packages\citycatio> + +Changes to: +1) run.py. a) different folder structure for parameter.csv and green_areas in DOCKER and NON DOCKER versions and b) xarrays ufunc changed to numpy c) rio.set.crs changed to rio.write.crs d) friction added to model call e) comment out figure creation near end f) read spatial rainfall if rainfall polygons exist. g) add rainfall_polygons to Model call h) read infiltration parameters if file exists and check if "value" exists in greenareas shape file if infiltration parameters exists i) add infiltration parameters to model call j) read reservoir data if it exists k) add reservoir to model call +2) utils.py modify for correct application of friction coeffs and sptial infiltration +3) friction.py allows friction coeffs (updated) +4) rainfall.py linetermination issues with different versions of the same package +5) rainfall_polygons.py change geoseries to GeoDataFrame +6) model.py a) add infiltration_parameters to init and configuration call b) a) add reservoir to init and write definitions +7) green_areas.py choice of green areas or spatial green areas +8) configuration.py a) allow infiltration parameters in init and write b) allow init_surface_water_elv in init and write +9) in inputs/_init_.py add reservoir.py +10) create reserovir.py + +python run.py + + + + +cmd in folder C:\Users\steve\Documents\CityCAT-dafni\global-flood-impacts-frictioncoeffs-0.0.1 +docker build -t friction-coeffs . + + +docker build -t Global Urban_Flooding:Friction-coeffs . +Run container in Docker Desktop. In Images find friction-coeffs. Click on the play button. Then "optional settings" and call "Container name" something like "friction-coeffs" +A new Container called "fraction-coeffs" is produced. +*** not needed as default **** Enivonment Variables DATA_PATH /data + +The Dockerfile spefies that this reads script.py into the src folder and the data into the /data folder in the container. then python script.py is run. + +The output is in the Docker container. to view output copy it to a local path +docker cp friction-coeffs:/data/outputs/FrictionCoeffs.txt ./FrictionCoeffs.txt + + +https://docs.docker.com/guides/walkthroughs/run-a-container/ +https://docs.docker.com/reference/cli/docker/container/cp/ + + +docker save -o friction-coeffs.tar friction-coeffs +compress friction-coeffs.tar to friction-coeffs.tar/gz using 7Zip add to achive + + + +set DATA_PATH=C:\Users\steve\Documents\citycat\CityCAT-FrictionCoeffs-Docker\data +set DATA_PATH=C:\Users\steve\Documents\citycat\CityCAT-SecondModel-Docker\data +set NUMBER_TEST=5 or Enivonment Variables NUMBER_TEST 5 +docker system prune -a + + +in run.py + + #filename=file_path[0].split("\") + filename=file_path[0].split("\\") + + + + +additional environment variables. The others come from the command line in: +https://github.com/OpenCLIM/citycat-dafni/tree/master + +NAME=test +OUTPUT_INTERVAL=3600 + +setenv.bat +set PYTHONUNBUFFERED=1 +set RAINFALL_MODE=total_depth +set SIZE=0.1 +set DURATION=1 +set POST_EVENT_DURATION=0 +set TOTAL_DEPTH=40 +set RETURN_PERIOD=100 +set X=258722 +set Y=665028 +set OPEN_BOUNDARIES=True +set PERMEABLE_AREAS=polygons +set ROOF_STORAGE=0 +set TIME_HORIZON=2050 +set DATA_PATH=C:\Users\steve\Documents\citycat-dafni-0.20.4\data +set NAME=test +set OUTPUT_INTERVAL=3600 + + +C:\Users\steve\anaconda3\envs\myenv\lib\site-packages\citycatio\inputs\rainfall.py:37: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead. + self.data.applymap(float_to_str).to_csv(f, sep=' ', header=False, line_terminator='\n') + + +In rainfall.py + self.data.applymap(float_to_str).to_csv(f, sep=' ', header=False, line_terminator='\n') + self.data.applymap(float_to_str).to_csv(f, sep=' ', header=False, lineterminator='\n') + + + +velocity = xr.ufuncs.sqrt(a.x_vel**2+a.y_vel**2).astype(np.float64) +Xarray’s ufuncs have been removed, now that they can be replaced by numpy’s ufuncs in all supported versions of numpy. + +#velocity = xr.ufuncs.sqrt(a.x_vel**2+a.y_vel**2).astype(np.float64) +velocity = np.sqrt(a.x_vel**2+a.y_vel**2).astype(np.float64) +max_velocity = max_velocity.where(np.isfinite(max_velocity), other=output.fill_value) +#max_velocity = max_velocity.where(xr.ufuncs.isfinite(max_velocity), other=output.fill_value) +#max_velocity.rio.set_crs('EPSG:27700') +max_velocity.rio.write_crs('EPSG:27700') +max_vd_product = max_vd_product.where(np.isfinite(max_vd_product), other=output.fill_value) +#max_vd_product = max_vd_product.where(xr.ufuncs.isfinite(max_vd_product), other=output.fill_value) +#max_vd_product.rio.set_crs('EPSG:27700') +max_vd_product.rio.write_crs('EPSG:27700') + + +C:\Users\steve\anaconda3\envs\myenv\lib\site-packages\citycatio\utils.py + +add + +def geoseries_with_value_to_string(geoseries: gpd.GeoSeries, value, index=False, index_first=True): + """GeoSeries to CityCAT string representation + + Args: + geoseries: Polygons to convert + value: Fraction coefficient value + index: Whether or not to include the index + index_first: Whether or not to place the index before the number of points + Returns: + s (str): String representation readable by CityCAT + + """ + assert (geoseries.geom_type == 'Polygon').all(), 'Geometries must be of type Polygon' + + s = '{}\n'.format(len(geoseries)) + + for idx, geometry in geoseries.items(): + if not index: + s += '{}'.format(len(geometry.exterior.coords)) + elif index_first: + s += '{} {}'.format(idx, len(geometry.exterior.coords)) + else: + s += '{} {}'.format(len(geometry.exterior.coords), value[idx]) + x, y = geometry.exterior.coords.xy + for x_val in x: + s += ' {}'.format(x_val) + for y_val in y: + s += ' {}'.format(y_val) + s += '\n' + + return s + +C:\Users\steve\anaconda3\envs\myenv\lib\site-packages\citycatio\inputs\friction.py + +from ..utils import geoseries_with_value_to_string + + def write(self, path): + with open(os.path.join(path, 'FrictionCoeffs.txt'), 'w') as f: + f.write(geoseries_with_value_to_string(self.data.geometry,self.data.Value,index=True, index_first=False)) + + +plus add frction stuff to run.py + + +myenv +C:\Users\steve\anaconda3\envs\myenv\Lib\site-packages\citycatio> + +# If a parameter.csv is available: read the variables from the document +if len(parameter_file) == 1 : + file_path = os.path.splitext(parameter_file[0]) + print('Filepath:',file_path) + #filename=file_path[0].split("\") + filename=file_path[0].split("\\") + print('Filename:',filename[-1]) + + + diff --git a/script.py b/script.py new file mode 100644 index 0000000..0cb7408 --- /dev/null +++ b/script.py @@ -0,0 +1,136 @@ +import geopandas as gpd +import pandas as pd +#import numpy as np +import os +#import shutil +from zipfile import ZipFile +from glob import glob +#import subprocess +import csv + +# Define Data Paths +data_path = os.getenv('DATA_PATH', '/data') +inputs_path = os.path.join(data_path,'inputs') + + +# Define Input Paths +boundary_path = os.path.join(inputs_path,'boundary') +vector_path = os.path.join(inputs_path, 'friction_coeffs') +parameters_path = os.path.join(inputs_path, 'parameters') + +# Define and Create Output Paths +outputs_path = os.path.join(data_path, 'outputs') +outputs_path_ = data_path + '/' + 'outputs' +if not os.path.exists(outputs_path): + os.mkdir(outputs_path_) +frictioncoeffs_path = os.path.join(outputs_path, 'friction_coeffs') +frictioncoeffs_path_ = outputs_path + '/' + 'friction_coeffs' +if not os.path.exists(frictioncoeffs_path): + os.mkdir(frictioncoeffs_path_) +# parameter_outputs_path = os.path.join(outputs_path, 'parameters') +# parameter_outputs_path_ = outputs_path + '/' + 'parameters' +# if not os.path.exists(parameter_outputs_path): +# os.mkdir(parameter_outputs_path_) + +# Look to see if a parameter file has been added +parameter_file = glob(parameters_path + "/*.csv", recursive = True) +print('parameter_file:', parameter_file) + +# Identify the EPSG projection code +if len(parameter_file) == 1 : + parameters = pd.read_csv(parameter_file[0]) + with open(parameter_file[0]) as file_obj: + reader_obj = csv.reader(file_obj) + for row in reader_obj: + try: + if row[0] == 'PROJECTION': + projection = row[1] + except: + continue +else: + projection = os.getenv('PROJECTION') + +print('projection:',projection) + +# Identify input polygons and shapes (boundary of city, and OS grid cell references) +boundary_1 = glob(boundary_path + "/*.*", recursive = True) +print('Boundary File:',boundary_1) + +# Read in the boundary +boundary = gpd.read_file(boundary_1[0]) + +# Check boundary crs matches the projection +if boundary.crs != projection: + boundary.to_crs(epsg=projection, inplace=True) + +print('boundary_crs:', boundary.crs) + +# Identify the name of the boundary file for the city name +file_path = os.path.splitext(boundary_1[0]) +print('File_path:',file_path) +print(os.name) +#code for file names is messy and needs to be done better +if os.name=='nt': + filename=file_path[0].split("\\") +else: + filename=file_path[0].split("/") +print('filename:',filename) +location = filename[-1] +print('Location:',location) + +# Identify if the buildings are saved in a zip file +friction_files_zip = glob(vector_path + "/*.zip", recursive = True) +print(friction_files_zip) + +# If yes, unzip the file (if the user has formatted the data correctly, this should reveal a .gpkg) +if len(friction_files_zip) != 0: + for i in range (0,len(friction_files_zip)): + print('zip file found') + with ZipFile(friction_files_zip[i],'r') as zip: + zip.extractall(vector_path) + +# Identify geopackages containing the polygons of the buildings +friction_files = glob(vector_path + "/*.gpkg", recursive = True) + +if len(friction_files) != 0: + # Create a list of all of the gpkgs to be merged + to_merge=[] + to_merge=['XX' for n in range(len(friction_files))] + for i in range (0,len(friction_files)): + file_path = os.path.splitext(friction_files[i]) + if os.name=='nt': + filename=file_path[0].split("\\") + else: + filename=file_path[0].split("/") + to_merge[i]=filename[-1]+'.gpkg' + + print('to_merge:',to_merge) + + # Create a geodatabase and merge the data from each gpkg together + all_friction = [] + all_friction=gpd.GeoDataFrame(all_friction) + for cell in to_merge: + #gdf = gpd.read_file('/data/inputs/friction_coeffs/%s' %cell) + gdf = gpd.read_file(vector_path +'/' + cell) + all_friction = pd.concat([gdf, all_friction],ignore_index=True) + + all_friction.to_crs(epsg=projection, inplace=True) + + clipped = gpd.clip(all_friction,boundary) + + permeable_areas = 'polygons' + + all_frictions = clipped.to_file(os.path.join(outputs_path,'all_frictioncoeffs.shp')) + all_frictions = gpd.read_file(os.path.join(outputs_path,'all_frictioncoeffs.shp')) + all_frictions = all_frictions.explode() + all_frictions.reset_index(inplace=True, drop=True) + all_frictions1 = all_frictions.to_file(os.path.join(frictioncoeffs_path, location + '.gpkg'),driver='GPKG',index=False) + + os.remove(os.path.join(outputs_path,'all_frictioncoeffs.shp')) + os.remove(os.path.join(outputs_path,'all_frictioncoeffs.cpg')) + os.remove(os.path.join(outputs_path,'all_frictioncoeffs.dbf')) + os.remove(os.path.join(outputs_path,'all_frictioncoeffs.prj')) + os.remove(os.path.join(outputs_path,'all_frictioncoeffs.shx')) + + +