diff --git a/gis_fillers/_version.py b/gis_fillers/_version.py index 6820f36..9b36b86 100644 --- a/gis_fillers/_version.py +++ b/gis_fillers/_version.py @@ -1 +1 @@ -__version__ = '0.0.10' +__version__ = "0.0.10" diff --git a/gis_fillers/database.py b/gis_fillers/database.py index feb8399..7302bf2 100644 --- a/gis_fillers/database.py +++ b/gis_fillers/database.py @@ -1,6 +1,5 @@ - import psycopg2 -from psycopg2 import extras,sql +from psycopg2 import extras, sql import os import copy import logging @@ -14,39 +13,46 @@ logger = logging.getLogger(__name__) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) logger.addHandler(ch) logger.setLevel(logging.INFO) try: - import psycopg2 - from psycopg2 import extras - from psycopg2.extensions import register_adapter, AsIs - register_adapter(np.float64, AsIs) - register_adapter(np.int64, AsIs) -except ImportError: - logger.info('Psycopg2 not installed, pip install psycopg2 (or binary-psycopg2) if you want to use a PostgreSQL DB') + import psycopg2 + from psycopg2 import extras + from psycopg2.extensions import register_adapter, AsIs + register_adapter(np.float64, AsIs) + register_adapter(np.int64, AsIs) +except ImportError: + logger.info( + "Psycopg2 not installed, pip install psycopg2 (or binary-psycopg2) if you want to use a PostgreSQL DB" + ) def split_sql_init(script): - lines = script.split('\n') - formatted = '\n'.join([l for l in lines if l[:2]!='--']) - return formatted.split(';')[:-1] + lines = script.split("\n") + formatted = "\n".join([l for l in lines if l[:2] != "--"]) + return formatted.split(";")[:-1] + class Database(TemplateDatabase): - """ - This class creates a database object with the main structure, with a few methods to manipulate it. - To fill it, fillers are used (see Filler class). - The object uses a specific data folder and a list of files used for the fillers, with name, keyword, and potential download link. (move to filler class?) - """ - - def clean_db(self,gis_data_stay=False,commit=True,extra_whitelist=[],**kwargs): - if gis_data_stay: - extra_whitelist += ['zones', - 'zone_parents', - 'zone_levels', - 'gis_data', - 'gis_types'] - TemplateDatabase.clean_db(self,commit=commit,extra_whitelist=extra_whitelist,**kwargs) + """ + This class creates a database object with the main structure, with a few methods to manipulate it. + To fill it, fillers are used (see Filler class). + The object uses a specific data folder and a list of files used for the fillers, with name, keyword, and potential download link. (move to filler class?) + """ + + def clean_db(self, gis_data_stay=False, commit=True, extra_whitelist=[], **kwargs): + if gis_data_stay: + extra_whitelist += [ + "zones", + "zone_parents", + "zone_levels", + "gis_data", + "gis_types", + ] + TemplateDatabase.clean_db( + self, commit=commit, extra_whitelist=extra_whitelist, **kwargs + ) diff --git a/gis_fillers/fillers/fillers.py b/gis_fillers/fillers/fillers.py index 93380db..5096a01 100644 --- a/gis_fillers/fillers/fillers.py +++ b/gis_fillers/fillers/fillers.py @@ -1,2 +1 @@ - from db_fillers import Filler diff --git a/gis_fillers/fillers/zones/countries.py b/gis_fillers/fillers/zones/countries.py index d9f9139..f1f137b 100644 --- a/gis_fillers/fillers/zones/countries.py +++ b/gis_fillers/fillers/zones/countries.py @@ -10,109 +10,138 @@ from .. import fillers - class CountriesFiller(fillers.Filler): - """ - Fills in countries GIS shapes in zaehlsprengel gis_type - """ - year_list = (2001,2006,2010,2013,2016,2020) - def __init__(self, - gis_info="https://gisco-services.ec.europa.eu/distribution/v2/countries/download/ref-countries-{YEAR}-01m.geojson.zip", - gis_info_name="ref-countries-{YEAR}-01m.geojson", - geojson_gis_info_name="ref-countries-{YEAR}-01m.geojson", - fullgeojson_gis_info_name="ref-countries-{YEAR}-01m.geojson/CNTR_RG_01M_{YEAR}_4326.geojson", - LBgeojson_gis_info_name="ref-countries-{YEAR}-01m.geojson/CNTR_LB_{YEAR}_4326.geojson", - include_austria=False, - force=False, - year=None, - **kwargs): - self.force = force - if year is None: - self.year = self.year_list[-1] - else: - self.year = year - self.gis_info = gis_info.format(YEAR=self.year) - self.gis_info_name = gis_info_name.format(YEAR=self.year) - self.geojson_gis_info_name = geojson_gis_info_name.format(YEAR=self.year) - self.fullgeojson_gis_info_name = fullgeojson_gis_info_name.format(YEAR=self.year) - self.LBgeojson_gis_info_name = LBgeojson_gis_info_name.format(YEAR=self.year) - self.gis_type = 'zaehlsprengel' - if self.year not in self.year_list: - self.logger.warning('Year {} may not be available for countries GIS data; available years should be:{}'.format(self.year,self.year_list)) - fillers.Filler.__init__(self,name='countries',**kwargs) + """ + Fills in countries GIS shapes in zaehlsprengel gis_type + """ + + year_list = (2001, 2006, 2010, 2013, 2016, 2020) + + def __init__( + self, + gis_info="https://gisco-services.ec.europa.eu/distribution/v2/countries/download/ref-countries-{YEAR}-01m.geojson.zip", + gis_info_name="ref-countries-{YEAR}-01m.geojson", + geojson_gis_info_name="ref-countries-{YEAR}-01m.geojson", + fullgeojson_gis_info_name="ref-countries-{YEAR}-01m.geojson/CNTR_RG_01M_{YEAR}_4326.geojson", + LBgeojson_gis_info_name="ref-countries-{YEAR}-01m.geojson/CNTR_LB_{YEAR}_4326.geojson", + include_austria=False, + force=False, + year=None, + **kwargs + ): + self.force = force + if year is None: + self.year = self.year_list[-1] + else: + self.year = year + self.gis_info = gis_info.format(YEAR=self.year) + self.gis_info_name = gis_info_name.format(YEAR=self.year) + self.geojson_gis_info_name = geojson_gis_info_name.format(YEAR=self.year) + self.fullgeojson_gis_info_name = fullgeojson_gis_info_name.format( + YEAR=self.year + ) + self.LBgeojson_gis_info_name = LBgeojson_gis_info_name.format(YEAR=self.year) + self.gis_type = "zaehlsprengel" + if self.year not in self.year_list: + self.logger.warning( + "Year {} may not be available for countries GIS data; available years should be:{}".format( + self.year, self.year_list + ) + ) + fillers.Filler.__init__(self, name="countries", **kwargs) - def prepare(self): - if self.data_folder is None: - self.data_folder = self.db.data_folder - data_folder = self.data_folder + def prepare(self): + if self.data_folder is None: + self.data_folder = self.db.data_folder + data_folder = self.data_folder - #create folder if needed - if not os.path.exists(data_folder): - os.makedirs(data_folder) + # create folder if needed + if not os.path.exists(data_folder): + os.makedirs(data_folder) - self.db.cursor.execute(''' + self.db.cursor.execute( + """ SELECT COUNT(*) FROM gis_data gd INNER JOIN zone_levels zl ON zl.name='country' AND gd.zone_level=zl.id INNER JOIN gis_types gt ON gd.gis_type=gt.id AND gt.name=%s - ;''',(self.gis_type,)) - query_ans = self.db.cursor.fetchone() - if query_ans is not None and query_ans[0] >= 2: - self.done = True - else: - #GIS info - if not os.path.exists(os.path.join(data_folder,self.gis_info_name)): - if not os.path.exists(os.path.join(data_folder,self.gis_info_name+'.zip')): - self.download(url=self.gis_info,destination=self.gis_info_name+'.zip') - # self.download(url=self.gis_info,destination=os.path.join(data_folder,self.gis_info_name+'.zip')) - self.logger.info('Unzipping {}'.format(self.gis_info_name+'.zip')) - self.unzip(orig_file=self.gis_info_name+'.zip',destination=self.gis_info_name) - # self.unzip(orig_file=os.path.join(data_folder,self.gis_info_name+'.zip'),destination=os.path.join(data_folder,self.gis_info_name)) + ;""", + (self.gis_type,), + ) + query_ans = self.db.cursor.fetchone() + if query_ans is not None and query_ans[0] >= 2: + self.done = True + else: + # GIS info + if not os.path.exists(os.path.join(data_folder, self.gis_info_name)): + if not os.path.exists( + os.path.join(data_folder, self.gis_info_name + ".zip") + ): + self.download( + url=self.gis_info, destination=self.gis_info_name + ".zip" + ) + # self.download(url=self.gis_info,destination=os.path.join(data_folder,self.gis_info_name+'.zip')) + self.logger.info("Unzipping {}".format(self.gis_info_name + ".zip")) + self.unzip( + orig_file=self.gis_info_name + ".zip", + destination=self.gis_info_name, + ) + # self.unzip(orig_file=os.path.join(data_folder,self.gis_info_name+'.zip'),destination=os.path.join(data_folder,self.gis_info_name)) - def apply(self): - #filling zones info at different levels - self.fill_countries() - #filling gis data info - self.fill_gis_countries_LB() - self.fill_gis_countries() + def apply(self): + # filling zones info at different levels + self.fill_countries() + # filling gis data info + self.fill_gis_countries_LB() + self.fill_gis_countries() - - def fill_countries(self,filename=None): - - self.logger.info('Filling countries') - if filename is None: - filename = self.LBgeojson_gis_info_name # for children classes - self.record_file(filename=filename,filecode='countries_geojsonLB') - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES('country','Country') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - zs_geo = json.load(f) - extras.execute_batch(self.db.cursor,'''INSERT INTO zones(code,name,level) + def fill_countries(self, filename=None): + self.logger.info("Filling countries") + if filename is None: + filename = self.LBgeojson_gis_info_name # for children classes + self.record_file(filename=filename, filecode="countries_geojsonLB") + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES('country','Country') ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + zs_geo = json.load(f) + extras.execute_batch( + self.db.cursor, + """INSERT INTO zones(code,name,level) VALUES(%(code)s, %(name)s, (SELECT id FROM zone_levels WHERE name='country')) - ON CONFLICT DO NOTHING;''',({'code':gj['id'],'name':gj['properties']['NAME_ENGL']} for gj in zs_geo['features'])) - self.db.connection.commit() - + ON CONFLICT DO NOTHING;""", + ( + {"code": gj["id"], "name": gj["properties"]["NAME_ENGL"]} + for gj in zs_geo["features"] + ), + ) + self.db.connection.commit() - def fill_gis_countries_LB(self,filename=None,gis_type=None): - ''' - ''' - if gis_type is None: - gis_type = self.gis_type - self.logger.info('Filling countries GIS centers') - self.db.cursor.execute('INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;',(gis_type,)) - self.db.connection.commit() - if filename is None: - filename = self.LBgeojson_gis_info_name # for children classes - self.record_file(filename=filename,filecode='countries_geojsonLB') - with open(os.path.join(self.data_folder,filename),'r') as f: - zs_geo = json.load(f) + def fill_gis_countries_LB(self, filename=None, gis_type=None): + """ """ + if gis_type is None: + gis_type = self.gis_type + self.logger.info("Filling countries GIS centers") + self.db.cursor.execute( + "INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;", + (gis_type,), + ) + self.db.connection.commit() + if filename is None: + filename = self.LBgeojson_gis_info_name # for children classes + self.record_file(filename=filename, filecode="countries_geojsonLB") + with open(os.path.join(self.data_folder, filename), "r") as f: + zs_geo = json.load(f) - self.db.connection.commit() - extras.execute_batch(self.db.cursor,'''INSERT INTO gis_data( + self.db.connection.commit() + extras.execute_batch( + self.db.cursor, + """INSERT INTO gis_data( zone_id, zone_level, -- geom, @@ -123,27 +152,51 @@ def fill_gis_countries_LB(self,filename=None,gis_type=None): ST_SetSRID(ST_GeomFromGeoJSON(%(geom)s),4326), (SELECT id FROM gis_types WHERE name=%(gis_type)s)) ON CONFLICT DO NOTHING - ;''',({'zone_code':gj['id'],'level':'country','geom':str(gj['geometry']),'gis_type':gis_type} for gj in zs_geo['features'])) - self.db.connection.commit() + ;""", + ( + { + "zone_code": gj["id"], + "level": "country", + "geom": str(gj["geometry"]), + "gis_type": gis_type, + } + for gj in zs_geo["features"] + ), + ) + self.db.connection.commit() - def fill_gis_countries(self,filename=None,gis_type=None): - ''' - ''' - if gis_type is None: - gis_type = self.gis_type - self.logger.info('Filling countries GIS') - self.db.cursor.execute('INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;',(gis_type,)) - self.db.connection.commit() - if filename is None: - filename = self.fullgeojson_gis_info_name # for children classes - self.record_file(filename=filename,filecode='countries_geojson') - with open(os.path.join(self.data_folder,filename),'r') as f: - zs_geo = json.load(f) + def fill_gis_countries(self, filename=None, gis_type=None): + """ """ + if gis_type is None: + gis_type = self.gis_type + self.logger.info("Filling countries GIS") + self.db.cursor.execute( + "INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;", + (gis_type,), + ) + self.db.connection.commit() + if filename is None: + filename = self.fullgeojson_gis_info_name # for children classes + self.record_file(filename=filename, filecode="countries_geojson") + with open(os.path.join(self.data_folder, filename), "r") as f: + zs_geo = json.load(f) - extras.execute_batch(self.db.cursor,'''UPDATE gis_data + extras.execute_batch( + self.db.cursor, + """UPDATE gis_data SET geom=ST_SetSRID(ST_GeomFromGeoJSON(%(geom)s),4326) WHERE zone_id=(SELECT id FROM zones WHERE code=%(code)s) AND zone_level=(SELECT id FROM zone_levels WHERE name=%(level)s) AND gis_type=(SELECT id FROM gis_types WHERE name=%(gis_type)s) - ;''',({'geom':str(gj['geometry']),'code':gj['id'],'gis_type':gis_type,'level':'country'} for gj in zs_geo['features'])) - self.db.connection.commit() + ;""", + ( + { + "geom": str(gj["geometry"]), + "code": gj["id"], + "gis_type": gis_type, + "level": "country", + } + for gj in zs_geo["features"] + ), + ) + self.db.connection.commit() diff --git a/gis_fillers/fillers/zones/generic.py b/gis_fillers/fillers/zones/generic.py index 1e65216..feb096c 100644 --- a/gis_fillers/fillers/zones/generic.py +++ b/gis_fillers/fillers/zones/generic.py @@ -11,99 +11,122 @@ from .. import fillers - class ZonesFiller(fillers.Filler): - """ - Fills in zones GIS shapes in a specified gis_type and with a specified zone_level - expected file: csv with geometries as explicit string, in SRID 4326 - """ - def __init__(self, - filepath, - zone_level, - zone_level_pretty=None, - columns={'name':1,'code':2,'geom':3}, - force=False, - header = False, - gis_type='zaehlsprengel', - **kwargs): - self.force = force - self.columns = copy.deepcopy(columns) - self.filepath = filepath - self.header = header - self.zone_level = zone_level - if zone_level_pretty is None: - self.zone_level_pretty = self.zone_level - else: - self.zone_level_pretty = zone_level_pretty - self.gis_type = gis_type - fillers.Filler.__init__(self,name=f'generic_{zone_level}',**kwargs) - - def prepare(self): - if self.data_folder is None: - self.data_folder = self.db.data_folder - data_folder = self.data_folder - - #create folder if needed - if not os.path.exists(data_folder): - os.makedirs(data_folder) - - self.db.cursor.execute(''' + """ + Fills in zones GIS shapes in a specified gis_type and with a specified zone_level + expected file: csv with geometries as explicit string, in SRID 4326 + """ + + def __init__( + self, + filepath, + zone_level, + zone_level_pretty=None, + columns={"name": 1, "code": 2, "geom": 3}, + force=False, + header=False, + gis_type="zaehlsprengel", + **kwargs, + ): + self.force = force + self.columns = copy.deepcopy(columns) + self.filepath = filepath + self.header = header + self.zone_level = zone_level + if zone_level_pretty is None: + self.zone_level_pretty = self.zone_level + else: + self.zone_level_pretty = zone_level_pretty + self.gis_type = gis_type + fillers.Filler.__init__(self, name=f"generic_{zone_level}", **kwargs) + + def prepare(self): + if self.data_folder is None: + self.data_folder = self.db.data_folder + data_folder = self.data_folder + + # create folder if needed + if not os.path.exists(data_folder): + os.makedirs(data_folder) + + self.db.cursor.execute( + """ SELECT COUNT(*) FROM gis_data gd INNER JOIN zone_levels zl ON zl.name=%(zone_level)s AND gd.zone_level=zl.id INNER JOIN gis_types gt ON gd.gis_type=gt.id AND gt.name=%(gis_type)s - ;''',{'zone_level':self.zone_level,'gis_type':self.gis_type,}) - query_ans = self.db.cursor.fetchone() - if query_ans is not None and query_ans[0] >= 2: - self.done = True - - def apply(self): - #filling zones info at different levels - self.fill_zones() - #filling gis data info - self.fill_gis() - - self.fill_zs_children() - - - def fill_zones(self,filename=None): - - self.logger.info(f'Filling {self.zone_level}') - if filename is None: - filename = self.filepath - self.record_file(filename=filename,filecode=f'zones_{self.zone_level}') - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES(%(zone_level)s,%(zl_pretty)s) ON CONFLICT DO NOTHING;''',{'zone_level':self.zone_level,'zl_pretty':self.zone_level_pretty}) - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f) - if self.header: - next(reader) - extras.execute_batch(self.db.cursor,'''INSERT INTO zones(id,code,name,level) + ;""", + { + "zone_level": self.zone_level, + "gis_type": self.gis_type, + }, + ) + query_ans = self.db.cursor.fetchone() + if query_ans is not None and query_ans[0] >= 2: + self.done = True + + def apply(self): + # filling zones info at different levels + self.fill_zones() + # filling gis data info + self.fill_gis() + + self.fill_zs_children() + + def fill_zones(self, filename=None): + self.logger.info(f"Filling {self.zone_level}") + if filename is None: + filename = self.filepath + self.record_file(filename=filename, filecode=f"zones_{self.zone_level}") + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES(%(zone_level)s,%(zl_pretty)s) ON CONFLICT DO NOTHING;""", + {"zone_level": self.zone_level, "zl_pretty": self.zone_level_pretty}, + ) + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f) + if self.header: + next(reader) + extras.execute_batch( + self.db.cursor, + """INSERT INTO zones(id,code,name,level) VALUES(%(code)s,%(code)s, %(name)s, (SELECT id FROM zone_levels WHERE name=%(zone_level)s)) - ON CONFLICT DO NOTHING;''',({'code':r[self.columns['code']],'name':r[self.columns['name']],'zone_level':self.zone_level} for r in reader)) - self.db.connection.commit() - - - def fill_gis(self,filename=None,gis_type=None): - ''' - ''' - if gis_type is None: - gis_type = self.gis_type - self.logger.info(f'Filling {self.zone_level} GIS') - self.db.cursor.execute('INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;',(gis_type,)) - self.db.connection.commit() - if filename is None: - filename = self.filepath # for children classes - self.record_file(filename=filename,filecode=self.zone_level) - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f) - if self.header: - next(reader) - extras.execute_batch(self.db.cursor,'''INSERT INTO gis_data( + ON CONFLICT DO NOTHING;""", + ( + { + "code": r[self.columns["code"]], + "name": r[self.columns["name"]], + "zone_level": self.zone_level, + } + for r in reader + ), + ) + self.db.connection.commit() + + def fill_gis(self, filename=None, gis_type=None): + """ """ + if gis_type is None: + gis_type = self.gis_type + self.logger.info(f"Filling {self.zone_level} GIS") + self.db.cursor.execute( + "INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;", + (gis_type,), + ) + self.db.connection.commit() + if filename is None: + filename = self.filepath # for children classes + self.record_file(filename=filename, filecode=self.zone_level) + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f) + if self.header: + next(reader) + extras.execute_batch( + self.db.cursor, + """INSERT INTO gis_data( zone_id, zone_level, geom, @@ -115,14 +138,24 @@ def fill_gis(self,filename=None,gis_type=None): ST_Centroid(ST_SetSRID(ST_GeomFromText(%(geom)s),4326)), (SELECT id FROM gis_types WHERE name=%(gis_type)s)) ON CONFLICT DO NOTHING - ;''',({'zone_code':r[self.columns['code']],'level':self.zone_level,'geom':r[self.columns['geom']],'gis_type':gis_type} for r in reader)) - self.db.connection.commit() - - - def fill_zs_children(self): - self.logger.info(f'Filling {self.zone_level} children') - - self.db.cursor.execute(''' + ;""", + ( + { + "zone_code": r[self.columns["code"]], + "level": self.zone_level, + "geom": r[self.columns["geom"]], + "gis_type": gis_type, + } + for r in reader + ), + ) + self.db.connection.commit() + + def fill_zs_children(self): + self.logger.info(f"Filling {self.zone_level} children") + + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child,share) ( SELECT gdp.zone_level,gdp.zone_id,gdc.zone_level,gdc.zone_id,ST_Area(ST_Intersection(gdc.geom,gdp.geom),false)/ST_Area(gdc.geom,false) AS share FROM zone_levels zlp @@ -140,5 +173,7 @@ def fill_zs_children(self): AND gdc.gis_type=gt.id AND ST_Intersects(gdc.geom,gdp.geom) ) ON CONFLICT DO NOTHING - ;''',{'gis_type':self.gis_type,'zone_level':self.zone_level}) - self.db.connection.commit() + ;""", + {"gis_type": self.gis_type, "zone_level": self.zone_level}, + ) + self.db.connection.commit() diff --git a/gis_fillers/fillers/zones/hexagons.py b/gis_fillers/fillers/zones/hexagons.py index eb55956..7c0ea97 100644 --- a/gis_fillers/fillers/zones/hexagons.py +++ b/gis_fillers/fillers/zones/hexagons.py @@ -18,28 +18,32 @@ class HexagonsFiller(fillers.Filler): - def __init__(self, - res=10, - target_zone='AT', - target_zone_level='country', - gis_type='zaehlsprengel', - buffer=True, - truncate_shapes=True, - **kwargs): - fillers.Filler.__init__(self,**kwargs) - self.res = res - self.buffer = buffer - self.buffer_meters = h3.edge_length(self.res,unit='m')*2. - self.truncate_shapes = truncate_shapes - self.target_zone = target_zone - self.target_zone_level = target_zone_level - self.zone_level = f'{target_zone_level}_{target_zone}_hexagons_{res}' - self.gis_type = gis_type - - def get_hexagons(self): - if not hasattr(self,'hexagons'): - - gdf_buffered = gpd.GeoDataFrame.from_postgis(con=self.db.connection,crs=4326,sql=''' + def __init__( + self, + res=10, + target_zone="AT", + target_zone_level="country", + gis_type="zaehlsprengel", + buffer=True, + truncate_shapes=True, + **kwargs, + ): + fillers.Filler.__init__(self, **kwargs) + self.res = res + self.buffer = buffer + self.buffer_meters = h3.edge_length(self.res, unit="m") * 2.0 + self.truncate_shapes = truncate_shapes + self.target_zone = target_zone + self.target_zone_level = target_zone_level + self.zone_level = f"{target_zone_level}_{target_zone}_hexagons_{res}" + self.gis_type = gis_type + + def get_hexagons(self): + if not hasattr(self, "hexagons"): + gdf_buffered = gpd.GeoDataFrame.from_postgis( + con=self.db.connection, + crs=4326, + sql=""" SELECT ST_Buffer(gd.geom::geography,%(buffer)s) AS geom FROM zone_levels zl INNER JOIN zones z ON zl.id=z.level AND COALESCE(z.code,z.id::text)=%(target_zone)s::text @@ -50,10 +54,19 @@ def get_hexagons(self): ON gd.gis_type=gt.id AND gd.zone_id=z.id AND gd.zone_level=z.level ; - ''',params={'target_zone':self.target_zone,'gis_type':self.gis_type,'target_zone_level':self.target_zone_level,'buffer':(self.buffer_meters if self.buffer else 0)}) - - - gdf = gpd.GeoDataFrame.from_postgis(con=self.db.connection,crs=4326,sql=''' + """, + params={ + "target_zone": self.target_zone, + "gis_type": self.gis_type, + "target_zone_level": self.target_zone_level, + "buffer": (self.buffer_meters if self.buffer else 0), + }, + ) + + gdf = gpd.GeoDataFrame.from_postgis( + con=self.db.connection, + crs=4326, + sql=""" SELECT gd.geom::geography AS geom FROM zone_levels zl INNER JOIN zones z ON zl.id=z.level AND COALESCE(z.code,z.id::text)=%(target_zone)s::text @@ -64,68 +77,101 @@ def get_hexagons(self): ON gd.gis_type=gt.id AND gd.zone_id=z.id AND gd.zone_level=z.level ; - ''',params={'target_zone':self.target_zone,'gis_type':self.gis_type,'target_zone_level':self.target_zone_level,'buffer':(self.buffer_meters if self.buffer else 0)}) - - - # Get union of the shape (whole US) - union_poly = unary_union(gdf_buffered.geometry) - if self.buffer: - orig_union_poly = unary_union(gdf.geometry) - else: - orig_union_poly = union_poly - - # Find the hexagons within the shape boundary using PolyFill - hex_list=[] - if hasattr(union_poly,'geoms'): - for n,g in enumerate(union_poly.geoms): - temp = mapping(g) - temp['coordinates']=[[[j[1],j[0]] for j in i] for i in temp['coordinates']] - hex_list.extend(h3.polyfill(temp,res=self.res)) - else: - temp = mapping(union_poly) - temp['coordinates']=[[[j[1],j[0]] for j in i] for i in temp['coordinates']] - hex_list.extend(h3.polyfill(temp,res=self.res)) - - # Create hexagon data frame - ans_hex = pd.DataFrame(hex_list,columns=["hex_id"]) - - # Create hexagon geometry and GeoDataFrame - polygons = [Polygon(h3.h3_to_geo_boundary(x, geo_json=True)) for x in ans_hex["hex_id"]] - polygons_truncated = [orig_union_poly.intersection(p) for p in polygons] - if self.truncate_shapes: - ans_hex['geometry'] = [(p if p.area>0 else None) for p in polygons_truncated] - else: - ans_hex['geometry'] = [(p if pt.area>0 else None) for pt,p in zip(polygons_truncated,polygons)] - ans_hex = ans_hex.dropna() - ans_hex = gpd.GeoDataFrame(ans_hex).set_index('hex_id') - self.hexagons = ans_hex - - def apply(self): - self.fill_hexagons() - self.fill_parents() - self.fill_children() - - def fill_hexagons(self): - self.get_hexagons() - hex_gdf = self.hexagons - self.db.cursor.execute(''' + """, + params={ + "target_zone": self.target_zone, + "gis_type": self.gis_type, + "target_zone_level": self.target_zone_level, + "buffer": (self.buffer_meters if self.buffer else 0), + }, + ) + + # Get union of the shape (whole US) + union_poly = unary_union(gdf_buffered.geometry) + if self.buffer: + orig_union_poly = unary_union(gdf.geometry) + else: + orig_union_poly = union_poly + + # Find the hexagons within the shape boundary using PolyFill + hex_list = [] + if hasattr(union_poly, "geoms"): + for n, g in enumerate(union_poly.geoms): + temp = mapping(g) + temp["coordinates"] = [ + [[j[1], j[0]] for j in i] for i in temp["coordinates"] + ] + hex_list.extend(h3.polyfill(temp, res=self.res)) + else: + temp = mapping(union_poly) + temp["coordinates"] = [ + [[j[1], j[0]] for j in i] for i in temp["coordinates"] + ] + hex_list.extend(h3.polyfill(temp, res=self.res)) + + # Create hexagon data frame + ans_hex = pd.DataFrame(hex_list, columns=["hex_id"]) + + # Create hexagon geometry and GeoDataFrame + polygons = [ + Polygon(h3.h3_to_geo_boundary(x, geo_json=True)) + for x in ans_hex["hex_id"] + ] + polygons_truncated = [orig_union_poly.intersection(p) for p in polygons] + if self.truncate_shapes: + ans_hex["geometry"] = [ + (p if p.area > 0 else None) for p in polygons_truncated + ] + else: + ans_hex["geometry"] = [ + (p if pt.area > 0 else None) + for pt, p in zip(polygons_truncated, polygons) + ] + ans_hex = ans_hex.dropna() + ans_hex = gpd.GeoDataFrame(ans_hex).set_index("hex_id") + self.hexagons = ans_hex + + def apply(self): + self.fill_hexagons() + self.fill_parents() + self.fill_children() + + def fill_hexagons(self): + self.get_hexagons() + hex_gdf = self.hexagons + self.db.cursor.execute( + """ INSERT INTO gis_types(name) SELECT %(gis_type)s ON CONFLICT DO NOTHING; - ''',{'gis_type':self.gis_type}) + """, + {"gis_type": self.gis_type}, + ) - self.db.cursor.execute(''' + self.db.cursor.execute( + """ INSERT INTO zone_levels(name) SELECT %(zone_level)s ON CONFLICT DO NOTHING; - ''',{'zone_level':self.zone_level}) + """, + {"zone_level": self.zone_level}, + ) - extras.execute_batch(self.db.cursor,''' + extras.execute_batch( + self.db.cursor, + """ INSERT INTO zones(code,name,level) VALUES(%(hex_id)s, %(hex_id)s, (SELECT id FROM zone_levels WHERE name=%(zone_level)s)) ON CONFLICT DO NOTHING; - ''',({'hex_id':a[0],'geom':a[1],'zone_level':self.zone_level} for a in hex_gdf.itertuples())) - extras.execute_batch(self.db.cursor,''' + """, + ( + {"hex_id": a[0], "geom": a[1], "zone_level": self.zone_level} + for a in hex_gdf.itertuples() + ), + ) + extras.execute_batch( + self.db.cursor, + """ INSERT INTO gis_data( zone_id, zone_level, @@ -138,13 +184,24 @@ def fill_hexagons(self): ST_Centroid(ST_SetSRID(ST_GeomFromWKB(%(geom)s::geometry),4326)), (SELECT id FROM gis_types WHERE name=%(gis_type)s)) ON CONFLICT DO NOTHING - ''',({'hex_id':a[0],'geom':a[1].wkb_hex,'zone_level':self.zone_level,'gis_type':self.gis_type} for a in hex_gdf.itertuples())) - self.db.connection.commit() - - def fill_parents(self): - self.logger.info(f'Filling {self.zone_level} parents') - - self.db.cursor.execute(''' + """, + ( + { + "hex_id": a[0], + "geom": a[1].wkb_hex, + "zone_level": self.zone_level, + "gis_type": self.gis_type, + } + for a in hex_gdf.itertuples() + ), + ) + self.db.connection.commit() + + def fill_parents(self): + self.logger.info(f"Filling {self.zone_level} parents") + + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child,share) ( SELECT gdp.zone_level,gdp.zone_id,gdc.zone_level,gdc.zone_id,ST_Area(ST_Intersection(gdc.geom,gdp.geom),false)/ST_Area(gdc.geom,false) AS share FROM zone_levels zlc @@ -159,13 +216,16 @@ def fill_parents(self): ON gdp.gis_type=gt.id AND ST_Intersects(gdc.geom,gdp.geom) ) ON CONFLICT DO NOTHING - ;''',{'gis_type':self.gis_type,'zone_level':self.zone_level}) - self.db.connection.commit() + ;""", + {"gis_type": self.gis_type, "zone_level": self.zone_level}, + ) + self.db.connection.commit() - def fill_children(self): - self.logger.info(f'Filling {self.zone_level} children') + def fill_children(self): + self.logger.info(f"Filling {self.zone_level} children") - self.db.cursor.execute(''' + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child,share) ( SELECT gdp.zone_level,gdp.zone_id,gdc.zone_level,gdc.zone_id,ST_Area(ST_Intersection(gdc.geom,gdp.geom),false)/ST_Area(gdc.geom,false) AS share FROM zone_levels zlp @@ -180,5 +240,7 @@ def fill_children(self): ON gdc.gis_type=gt.id AND ST_Intersects(gdc.geom,gdp.geom) ) ON CONFLICT DO NOTHING - ;''',{'gis_type':self.gis_type,'zone_level':self.zone_level}) - self.db.connection.commit() + ;""", + {"gis_type": self.gis_type, "zone_level": self.zone_level}, + ) + self.db.connection.commit() diff --git a/gis_fillers/fillers/zones/zaehlsprengel.py b/gis_fillers/fillers/zones/zaehlsprengel.py index 1f35bb7..15b7955 100644 --- a/gis_fillers/fillers/zones/zaehlsprengel.py +++ b/gis_fillers/fillers/zones/zaehlsprengel.py @@ -13,179 +13,218 @@ from .. import fillers - class ZaehlsprengelFiller(fillers.Filler): - """ - This class fills in geographical data for Austria, with structure: - zaehlsprengel < gemeinde < bezirk < bundesland - Source is from STAT austria, in three files: - GIS info: http://data.statistik.gv.at/data/OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101.zip - population info: https://statistik.at/fileadmin/pages/405/Bevoelkerung_am_1.1.{YEAR}_nach_Zaehlsprengel__Gebietsstand_1.1.{YEAR}_.ods - bezirk names: http://www.statistik.at/verzeichnis/reglisten/polbezirke.csv - - Be attentive to the issue year of the different sources, they need to match (typically GIS info is ahead one year if you take the latest). - - The simplified attribute is used to tell the filler to preprocess the shapefile and simplify the edges with mapshaper/topojson - """ - - def __init__(self, - gis_info="https://data.statistik.gv.at/data/OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101.zip", - gis_info_name="OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101", - gis_info_fullname="OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101/STATISTIK_AUSTRIA_ZSP_{YEAR}0101.shp", - geojson_gis_info_name="STATISTIK_AUSTRIA_ZSP_{YEAR}0101.geojson", - pop_info="https://www.statistik.at/fileadmin/pages/405/Bev_{YEAR}_Zaehlsprengel.ods", - # pop_info="https://statistik.at/fileadmin/pages/405/Bevoelkerung_am_01.01.{YEAR}_nach_Zaehlsprengel__Gebietsstand_01.01.{YEAR}_.ods", - # pop_info_name='einwohnerzahl_nach_zaehlsprengel_1.1.2020_gebietsstand_1.1.2020', - bezirk_info='https://www.statistik.at/verzeichnis/reglisten/polbezirke.csv', - bezirk_info_name='polbezirke.csv', - simplified=False, - include_population=True, - force=False, - remove_bz_900=True, - year=2023, - simplify_engine='topojson', - **kwargs): - self.force = force - self.year = year - self.gis_info = gis_info.format(YEAR=self.year) - self.gis_info_name = gis_info_name.format(YEAR=self.year) - self.gis_info_fullname = gis_info_fullname.format(YEAR=self.year) - self.geojson_gis_info_name = geojson_gis_info_name.format(YEAR=self.year) - self.pop_info = pop_info.format(YEAR=self.year) - self.pop_info_name = '.'.join(self.pop_info.split('/')[-1].split('.')[:-1]) - self.bezirk_info = bezirk_info - self.bezirk_info_name = bezirk_info_name - self.simplified = simplified - self.remove_bz_900 = remove_bz_900 - self.include_population = include_population - if self.simplified: - self.gis_type = 'zaehlsprengel_simplified' - self.simplify_engine = simplify_engine - if self.simplify_engine == 'mapshaper': - try: - subprocess.check_output('mapshaper --version'.split(' ')) - except FileNotFoundError: - raise FileNotFoundError('Mapshaper is not installed, please install for node.js with: npm install -g mapshaper') - else: - self.gis_type = 'zaehlsprengel' - fillers.Filler.__init__(self,name=self.gis_type,**kwargs) - - def prepare(self): - if self.data_folder is None: - self.data_folder = self.db.data_folder - data_folder = self.data_folder - - #create folder if needed - if not os.path.exists(data_folder): - os.makedirs(data_folder) - - self.db.cursor.execute(''' + """ + This class fills in geographical data for Austria, with structure: + zaehlsprengel < gemeinde < bezirk < bundesland + Source is from STAT austria, in three files: + GIS info: http://data.statistik.gv.at/data/OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101.zip + population info: https://statistik.at/fileadmin/pages/405/Bevoelkerung_am_1.1.{YEAR}_nach_Zaehlsprengel__Gebietsstand_1.1.{YEAR}_.ods + bezirk names: http://www.statistik.at/verzeichnis/reglisten/polbezirke.csv + + Be attentive to the issue year of the different sources, they need to match (typically GIS info is ahead one year if you take the latest). + + The simplified attribute is used to tell the filler to preprocess the shapefile and simplify the edges with mapshaper/topojson + """ + + def __init__( + self, + gis_info="https://data.statistik.gv.at/data/OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101.zip", + gis_info_name="OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101", + gis_info_fullname="OGDEXT_ZSP_1_STATISTIK_AUSTRIA_{YEAR}0101/STATISTIK_AUSTRIA_ZSP_{YEAR}0101.shp", + geojson_gis_info_name="STATISTIK_AUSTRIA_ZSP_{YEAR}0101.geojson", + pop_info="https://www.statistik.at/fileadmin/pages/405/Bev_{YEAR}_Zaehlsprengel.ods", + # pop_info="https://statistik.at/fileadmin/pages/405/Bevoelkerung_am_01.01.{YEAR}_nach_Zaehlsprengel__Gebietsstand_01.01.{YEAR}_.ods", + # pop_info_name='einwohnerzahl_nach_zaehlsprengel_1.1.2020_gebietsstand_1.1.2020', + bezirk_info="https://www.statistik.at/verzeichnis/reglisten/polbezirke.csv", + bezirk_info_name="polbezirke.csv", + simplified=False, + include_population=True, + force=False, + remove_bz_900=True, + year=2023, + simplify_engine="topojson", + **kwargs, + ): + self.force = force + self.year = year + self.gis_info = gis_info.format(YEAR=self.year) + self.gis_info_name = gis_info_name.format(YEAR=self.year) + self.gis_info_fullname = gis_info_fullname.format(YEAR=self.year) + self.geojson_gis_info_name = geojson_gis_info_name.format(YEAR=self.year) + self.pop_info = pop_info.format(YEAR=self.year) + self.pop_info_name = ".".join(self.pop_info.split("/")[-1].split(".")[:-1]) + self.bezirk_info = bezirk_info + self.bezirk_info_name = bezirk_info_name + self.simplified = simplified + self.remove_bz_900 = remove_bz_900 + self.include_population = include_population + if self.simplified: + self.gis_type = "zaehlsprengel_simplified" + self.simplify_engine = simplify_engine + if self.simplify_engine == "mapshaper": + try: + subprocess.check_output("mapshaper --version".split(" ")) + except FileNotFoundError: + raise FileNotFoundError( + "Mapshaper is not installed, please install for node.js with: npm install -g mapshaper" + ) + else: + self.gis_type = "zaehlsprengel" + fillers.Filler.__init__(self, name=self.gis_type, **kwargs) + + def prepare(self): + if self.data_folder is None: + self.data_folder = self.db.data_folder + data_folder = self.data_folder + + # create folder if needed + if not os.path.exists(data_folder): + os.makedirs(data_folder) + + self.db.cursor.execute( + """ SELECT 1 FROM gis_data gd INNER JOIN zone_levels zl ON zl.name='zaehlsprengel' AND gd.zone_level=zl.id INNER JOIN gis_types gt ON gd.gis_type=gt.id AND gt.name=%s - ;''',(self.gis_type,)) - if self.db.cursor.fetchone() is not None and not self.force: - self.done = True - else: - #GIS info - if not os.path.exists(os.path.join(data_folder,self.gis_info_name)): - if not os.path.exists(os.path.join(data_folder,self.gis_info_name+'.zip')): - self.download(url=self.gis_info,destination=self.gis_info_name+'.zip') - self.logger.info('Unzipping {}'.format(self.gis_info_name+'.zip')) - self.unzip(orig_file=self.gis_info_name+'.zip',destination=self.gis_info_name) - - #Simplifying shapefile into geojson - if self.simplified and not os.path.exists(os.path.join(self.data_folder,self.geojson_gis_info_name)): - self.logger.info('Converting Shapefile into GeoJSON with less edges') - self.simplify_shapefile() - - #bezirk info - if not os.path.exists(os.path.join(data_folder,self.bezirk_info_name)): - self.download(url=self.bezirk_info,destination=self.bezirk_info_name) - - #pop_info - file_ext = self.pop_info.split('.')[-1] - if not os.path.exists(os.path.join(data_folder,self.pop_info_name+'.csv')): - if not os.path.exists(os.path.join(data_folder,self.pop_info_name+'.'+file_ext)): - self.logger.info('Downloading {}'.format(self.pop_info)) - self.download(url=self.pop_info,destination=self.pop_info_name+'.'+file_ext,wget=True) - # raise NotImplementedError('Complex JS query pattern, please download manually') - self.convert_spreadsheet(orig_file=self.pop_info_name+'.'+file_ext,destination=self.pop_info_name+'.csv') - - def apply(self): - #filling zones info at different levels - self.fill_zs() - self.fill_gemeinde() - self.fill_bezirk() - self.fill_bundesland() - self.fill_country() - #filling parenthood between levels - self.fill_parents_zs_g() - self.fill_parents_zs_bz() - self.fill_parents_zs_bl() - self.fill_parents_g_bz() - self.fill_parents_g_bl() - self.fill_parents_bz_bl() - self.fill_parents_country() - #filling gis data info - self.fill_gis_zs() - self.fill_gis_g() - self.fill_gis_bz() - self.fill_gis_bl() - self.fill_gis_country() - #filling population data - if self.include_population: - self.fill_population() - - #@check_empty(table='zones') - def fill_zs(self,filename=None): - self.logger.info('Filling zaehlsprengel') - if filename is None: - filename = self.pop_info_name+'.csv' - self.record_file(filename=filename,filecode='zaehlsprengel') - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES('zaehlsprengel','Zählsprengel') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f) - next(reader) #remove header - ans = [r for r in reader] - try: - int(ans[0][3]) - except ValueError: - ans = ans[1:] - ans = self.clean_reader(ans) # two last lines are just empty/info - extras.execute_batch(self.db.cursor,'''INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='zaehlsprengel')) ON CONFLICT DO NOTHING;''',((int(r[3]),r[4]) for r in ans)) - self.db.connection.commit() - - def fill_population(self,filename=None): - self.logger.info('Filling population data') - if filename is None: - filename = self.pop_info_name+'.csv' - self.db.cursor.execute('''INSERT INTO zone_attribute_types(name) VALUES('zs_population') ON CONFLICT DO NOTHING;''') - #self.db.cursor.execute('''INSERT INTO scenarios(name) VALUES('nothing') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f) - next(reader) #remove header - ans = [r for r in reader] - try: - int(ans[0][3]) - except ValueError: - ans = ans[1:] - - ans = self.clean_reader(ans) # two last lines are just empty/info - extras.execute_batch(self.db.cursor,'''INSERT INTO zone_attributes(zone,zone_level,attribute,int_value)--,scenario) + ;""", + (self.gis_type,), + ) + if self.db.cursor.fetchone() is not None and not self.force: + self.done = True + else: + # GIS info + if not os.path.exists(os.path.join(data_folder, self.gis_info_name)): + if not os.path.exists( + os.path.join(data_folder, self.gis_info_name + ".zip") + ): + self.download( + url=self.gis_info, destination=self.gis_info_name + ".zip" + ) + self.logger.info("Unzipping {}".format(self.gis_info_name + ".zip")) + self.unzip( + orig_file=self.gis_info_name + ".zip", + destination=self.gis_info_name, + ) + + # Simplifying shapefile into geojson + if self.simplified and not os.path.exists( + os.path.join(self.data_folder, self.geojson_gis_info_name) + ): + self.logger.info("Converting Shapefile into GeoJSON with less edges") + self.simplify_shapefile() + + # bezirk info + if not os.path.exists(os.path.join(data_folder, self.bezirk_info_name)): + self.download(url=self.bezirk_info, destination=self.bezirk_info_name) + + # pop_info + file_ext = self.pop_info.split(".")[-1] + if not os.path.exists( + os.path.join(data_folder, self.pop_info_name + ".csv") + ): + if not os.path.exists( + os.path.join(data_folder, self.pop_info_name + "." + file_ext) + ): + self.logger.info("Downloading {}".format(self.pop_info)) + self.download( + url=self.pop_info, + destination=self.pop_info_name + "." + file_ext, + wget=True, + ) + # raise NotImplementedError('Complex JS query pattern, please download manually') + self.convert_spreadsheet( + orig_file=self.pop_info_name + "." + file_ext, + destination=self.pop_info_name + ".csv", + ) + + def apply(self): + # filling zones info at different levels + self.fill_zs() + self.fill_gemeinde() + self.fill_bezirk() + self.fill_bundesland() + self.fill_country() + # filling parenthood between levels + self.fill_parents_zs_g() + self.fill_parents_zs_bz() + self.fill_parents_zs_bl() + self.fill_parents_g_bz() + self.fill_parents_g_bl() + self.fill_parents_bz_bl() + self.fill_parents_country() + # filling gis data info + self.fill_gis_zs() + self.fill_gis_g() + self.fill_gis_bz() + self.fill_gis_bl() + self.fill_gis_country() + # filling population data + if self.include_population: + self.fill_population() + + # @check_empty(table='zones') + def fill_zs(self, filename=None): + self.logger.info("Filling zaehlsprengel") + if filename is None: + filename = self.pop_info_name + ".csv" + self.record_file(filename=filename, filecode="zaehlsprengel") + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES('zaehlsprengel','Zählsprengel') ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f) + next(reader) # remove header + ans = [r for r in reader] + try: + int(ans[0][3]) + except ValueError: + ans = ans[1:] + ans = self.clean_reader(ans) # two last lines are just empty/info + extras.execute_batch( + self.db.cursor, + """INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='zaehlsprengel')) ON CONFLICT DO NOTHING;""", + ((int(r[3]), r[4]) for r in ans), + ) + self.db.connection.commit() + + def fill_population(self, filename=None): + self.logger.info("Filling population data") + if filename is None: + filename = self.pop_info_name + ".csv" + self.db.cursor.execute( + """INSERT INTO zone_attribute_types(name) VALUES('zs_population') ON CONFLICT DO NOTHING;""" + ) + # self.db.cursor.execute('''INSERT INTO scenarios(name) VALUES('nothing') ON CONFLICT DO NOTHING;''') + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f) + next(reader) # remove header + ans = [r for r in reader] + try: + int(ans[0][3]) + except ValueError: + ans = ans[1:] + + ans = self.clean_reader(ans) # two last lines are just empty/info + extras.execute_batch( + self.db.cursor, + """INSERT INTO zone_attributes(zone,zone_level,attribute,int_value)--,scenario) SELECT %s, zl.id, zat.id,%s--,s.id FROM zone_levels zl INNER JOIN zone_attribute_types zat ON zl.name='zaehlsprengel' AND zat.name='zs_population' --INNER JOIN scenarios s --ON s.name='nothing' - ON CONFLICT DO NOTHING;''',((int(r[3]),r[5]) for r in ans)) - self.db.cursor.execute(''' + ON CONFLICT DO NOTHING;""", + ((int(r[3]), r[5]) for r in ans), + ) + self.db.cursor.execute( + """ INSERT INTO zone_attributes(zone,zone_level,attribute,int_value)--,scenario) SELECT z.id, z.level, zat.id,SUM(za.int_value)--,s.id FROM zones z @@ -202,87 +241,115 @@ def fill_population(self,filename=None): AND za.attribute=zat.id GROUP BY z.id, z.level,zat.id--,s.id ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def simplify_shapefile(self,**kwargs): - if self.simplify_engine == 'mapshaper': - self.simplify_shapefile_mapshaper(**kwargs) - elif self.simplify_engine == 'topojson': - self.simplify_shapefile_topojson(**kwargs) - else: - raise NotImplementedError(f'Simplifying engine should be mapshaper or topojson. Not implemented: {self.simplify_engine}') - - def simplify_shapefile_topojson(self,input_path=None,output_path=None,simplify_param=10): - - if input_path is None: - input_path = os.path.join(self.data_folder,self.gis_info_fullname) - if output_path is None: - output_path = os.path.join(self.data_folder,self.geojson_gis_info_name) - gdf = gpd.read_file(input_path).to_crs(4326) - topo = tp.Topology(gdf) - del gdf - topo.toposimplify(simplify_param).to_geojson(fp=output_path) - del topo - # with open(output_path,'w') as f: - # f.write(output) - # del output - - def simplify_shapefile_mapshaper(self,input_path=None,output_path=None,method='visvalingam',percentage=0.2,interval=None,weight=0.5): - ''' - Simplifying the shapefile - Parameter choice is not really implemented in this class; in this case the name of the gis_type attribute used in the database should integrate them to avoid confusion - ''' - if input_path is None: - input_path = os.path.join(self.data_folder,self.gis_info_fullname) - if output_path is None: - output_path = os.path.join(self.data_folder,self.geojson_gis_info_name) - if method == 'dp': - options = ' dp' - elif method == 'visvalingam': - options = 'visvalingam' - else: - raise NotImplementedError('method should be dp or visvalingam, not '+str(method)) - if weight is not None: - options += ' weighted weighting={}'.format(weight) - if percentage is not None: - options += ' percentage={}'.format(percentage) - if interval is not None: - options += ' interval={}'.format(interval) - cmd = 'mapshaper {input_path} -simplify {options} keep-shapes -proj wgs84 -o {output_path}'.format(input_path=input_path,output_path=output_path,options=options) - self.logger.info('+ '+cmd) - cmd_output = subprocess.check_output(cmd.split(' ')) - self.logger.info(cmd_output) - - def clean_reader(self,reader): - ans = reader - while ans[-1]=='' or ans[-1][0].startswith('Q: STATISTIK AUSTRIA, Statistik des Bevölkerungsstandes.') or ans[-1][0].startswith('"Q: STATISTIK AUSTRIA, Statistik des Bevölkerungsstandes.'): - ans.pop(-1) - # two last lines are just empty/info - return ans - - - def fill_gis_zs(self,filename=None,gis_type=None): - ''' - distinguishing between raw shapefile (original highly detailed geoms), or processed geojsonfile (simplified via mapshaper) - ''' - if self.simplified: - filetype = 'geojson' - else: - filetype = 'shapefile' - - if gis_type is None: - gis_type = self.gis_type - self.logger.info('Filling zaehlsprengel GIS') - self.db.cursor.execute('INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;',(gis_type,)) - self.db.connection.commit() - if filetype == 'geojson': - if filename is None: - filename = self.geojson_gis_info_name # for children classes - self.record_file(filename=filename,filecode='zaehlsprengel_geojson') - with open(os.path.join(self.data_folder,filename),'r') as f: - zs_geo = json.load(f) - extras.execute_batch(self.db.cursor,''' + ;""" + ) + self.db.connection.commit() + + def simplify_shapefile(self, **kwargs): + if self.simplify_engine == "mapshaper": + self.simplify_shapefile_mapshaper(**kwargs) + elif self.simplify_engine == "topojson": + self.simplify_shapefile_topojson(**kwargs) + else: + raise NotImplementedError( + f"Simplifying engine should be mapshaper or topojson. Not implemented: {self.simplify_engine}" + ) + + def simplify_shapefile_topojson( + self, input_path=None, output_path=None, simplify_param=10 + ): + if input_path is None: + input_path = os.path.join(self.data_folder, self.gis_info_fullname) + if output_path is None: + output_path = os.path.join(self.data_folder, self.geojson_gis_info_name) + gdf = gpd.read_file(input_path).to_crs(4326) + topo = tp.Topology(gdf) + del gdf + topo.toposimplify(simplify_param).to_geojson(fp=output_path) + del topo + # with open(output_path,'w') as f: + # f.write(output) + # del output + + def simplify_shapefile_mapshaper( + self, + input_path=None, + output_path=None, + method="visvalingam", + percentage=0.2, + interval=None, + weight=0.5, + ): + """ + Simplifying the shapefile + Parameter choice is not really implemented in this class; in this case the name of the gis_type attribute used in the database should integrate them to avoid confusion + """ + if input_path is None: + input_path = os.path.join(self.data_folder, self.gis_info_fullname) + if output_path is None: + output_path = os.path.join(self.data_folder, self.geojson_gis_info_name) + if method == "dp": + options = " dp" + elif method == "visvalingam": + options = "visvalingam" + else: + raise NotImplementedError( + "method should be dp or visvalingam, not " + str(method) + ) + if weight is not None: + options += " weighted weighting={}".format(weight) + if percentage is not None: + options += " percentage={}".format(percentage) + if interval is not None: + options += " interval={}".format(interval) + cmd = "mapshaper {input_path} -simplify {options} keep-shapes -proj wgs84 -o {output_path}".format( + input_path=input_path, output_path=output_path, options=options + ) + self.logger.info("+ " + cmd) + cmd_output = subprocess.check_output(cmd.split(" ")) + self.logger.info(cmd_output) + + def clean_reader(self, reader): + ans = reader + while ( + ans[-1] == "" + or ans[-1][0].startswith( + "Q: STATISTIK AUSTRIA, Statistik des Bevölkerungsstandes." + ) + or ans[-1][0].startswith( + '"Q: STATISTIK AUSTRIA, Statistik des Bevölkerungsstandes.' + ) + ): + ans.pop(-1) + # two last lines are just empty/info + return ans + + def fill_gis_zs(self, filename=None, gis_type=None): + """ + distinguishing between raw shapefile (original highly detailed geoms), or processed geojsonfile (simplified via mapshaper) + """ + if self.simplified: + filetype = "geojson" + else: + filetype = "shapefile" + + if gis_type is None: + gis_type = self.gis_type + self.logger.info("Filling zaehlsprengel GIS") + self.db.cursor.execute( + "INSERT INTO gis_types(name) VALUES(%s) ON CONFLICT DO NOTHING;", + (gis_type,), + ) + self.db.connection.commit() + if filetype == "geojson": + if filename is None: + filename = self.geojson_gis_info_name # for children classes + self.record_file(filename=filename, filecode="zaehlsprengel_geojson") + with open(os.path.join(self.data_folder, filename), "r") as f: + zs_geo = json.load(f) + extras.execute_batch( + self.db.cursor, + """ INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) VALUES (%s, (SELECT id FROM zone_levels WHERE name=%s), @@ -290,88 +357,140 @@ def fill_gis_zs(self,filename=None,gis_type=None): ST_SetSRID(ST_Centroid(ST_MakeValid(ST_GeomFromGeoJSON(%s))),4326), (SELECT id FROM gis_types WHERE name=%s)) ON CONFLICT DO NOTHING - ;''',((int(gj['properties'][('id' if 'id' in gj['properties'].keys() else 'g_id')]), - 'zaehlsprengel', - str(gj['geometry']), - str(gj['geometry']), - gis_type) for gj in zs_geo['features'])) - elif filetype == 'shapefile': - if filename is None: - filename = self.gis_info_fullname - self.record_file(filename=filename,filecode='zaehlsprengel_shapefile') - with shapefile.Reader(os.path.join(self.data_folder,filename)) as sf: - extras.execute_batch(self.db.cursor,'''INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) VALUES (%s,(SELECT id FROM zone_levels WHERE name=%s),ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),31287),4326),ST_Centroid(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),31287),4326)),(SELECT id FROM gis_types WHERE name=%s)) ON CONFLICT DO NOTHING;''',((int(r[0]),'zaehlsprengel',json.dumps(s.__geo_interface__),json.dumps(s.__geo_interface__),gis_type) for s,r in zip(sf.shapes(),sf.records()))) - else: - raise ValueError('ZS filetype unknown:',filetype) - self.db.connection.commit() - def fill_gemeinde(self,filename=None): - self.logger.info('Filling Gemeinde') - if filename is None: - filename = self.pop_info_name+'.csv' - self.record_file(filename=filename,filecode='zaehlsprengel') - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES('gemeinde','Gemeinde') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f) - next(reader) #remove header - ans = [r for r in reader] - try: - int(ans[0][3]) - except ValueError: - ans = ans[1:] - ans = self.clean_reader(ans) - - extras.execute_batch(self.db.cursor,'''INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='gemeinde')) ON CONFLICT DO NOTHING;''',((int(r[1]),r[2]) for r in ans)) - self.db.connection.commit() - - def fill_bezirk(self,filename=None): - self.logger.info('Filling Bezirk') - - if filename is None: - filename = self.bezirk_info_name - self.record_file(filename=filename,filecode='bezirk') - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES('bezirk','Bezirk') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f,delimiter=';') - next(reader) - next(reader) - next(reader) - ans = [r for r in reader] - ans.pop(-1) # last line and 3 first lines are just empty/info - extras.execute_batch(self.db.cursor,'''INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='bezirk')) ON CONFLICT DO NOTHING;''',((int(r[4]),r[3]) for r in ans if (not self.remove_bz_900 or int(r[4])!=900))) - self.db.connection.commit() - - def fill_bundesland(self,filename=None): - self.logger.info('Filling Bundesland') - - if filename is None: - filename = self.bezirk_info_name - self.record_file(filename=filename,filecode='bezirk') - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES('bundesland','Bundesland') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f,delimiter=';') - next(reader) - next(reader) - next(reader) - ans = [r for r in reader] - ans.pop(-1) # last line and 3 first lines are just empty/info - extras.execute_batch(self.db.cursor,'''INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='bundesland')) ON CONFLICT DO NOTHING;''',((int(r[0]),r[1]) for r in ans)) - self.db.connection.commit() - - def fill_country(self): - self.logger.info('Filling Country') - - self.db.cursor.execute('''INSERT INTO zone_levels(name,pretty_name) VALUES('country','Country') ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - self.db.cursor.execute('''INSERT INTO zones(name,code,level) VALUES('Österreich','AT',(SELECT id FROM zone_levels WHERE name='country')) ON CONFLICT DO NOTHING;''') - self.db.connection.commit() - - def fill_parents_zs_g(self): - self.logger.info('Filling zaehlsprengel gemeinde parents') - - self.db.cursor.execute(''' + ;""", + ( + ( + int( + gj["properties"][ + ("id" if "id" in gj["properties"].keys() else "g_id") + ] + ), + "zaehlsprengel", + str(gj["geometry"]), + str(gj["geometry"]), + gis_type, + ) + for gj in zs_geo["features"] + ), + ) + elif filetype == "shapefile": + if filename is None: + filename = self.gis_info_fullname + self.record_file(filename=filename, filecode="zaehlsprengel_shapefile") + with shapefile.Reader(os.path.join(self.data_folder, filename)) as sf: + extras.execute_batch( + self.db.cursor, + """INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) VALUES (%s,(SELECT id FROM zone_levels WHERE name=%s),ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),31287),4326),ST_Centroid(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(%s),31287),4326)),(SELECT id FROM gis_types WHERE name=%s)) ON CONFLICT DO NOTHING;""", + ( + ( + int(r[0]), + "zaehlsprengel", + json.dumps(s.__geo_interface__), + json.dumps(s.__geo_interface__), + gis_type, + ) + for s, r in zip(sf.shapes(), sf.records()) + ), + ) + else: + raise ValueError("ZS filetype unknown:", filetype) + self.db.connection.commit() + + def fill_gemeinde(self, filename=None): + self.logger.info("Filling Gemeinde") + if filename is None: + filename = self.pop_info_name + ".csv" + self.record_file(filename=filename, filecode="zaehlsprengel") + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES('gemeinde','Gemeinde') ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f) + next(reader) # remove header + ans = [r for r in reader] + try: + int(ans[0][3]) + except ValueError: + ans = ans[1:] + ans = self.clean_reader(ans) + + extras.execute_batch( + self.db.cursor, + """INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='gemeinde')) ON CONFLICT DO NOTHING;""", + ((int(r[1]), r[2]) for r in ans), + ) + self.db.connection.commit() + + def fill_bezirk(self, filename=None): + self.logger.info("Filling Bezirk") + + if filename is None: + filename = self.bezirk_info_name + self.record_file(filename=filename, filecode="bezirk") + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES('bezirk','Bezirk') ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f, delimiter=";") + next(reader) + next(reader) + next(reader) + ans = [r for r in reader] + ans.pop(-1) # last line and 3 first lines are just empty/info + extras.execute_batch( + self.db.cursor, + """INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='bezirk')) ON CONFLICT DO NOTHING;""", + ( + (int(r[4]), r[3]) + for r in ans + if (not self.remove_bz_900 or int(r[4]) != 900) + ), + ) + self.db.connection.commit() + + def fill_bundesland(self, filename=None): + self.logger.info("Filling Bundesland") + + if filename is None: + filename = self.bezirk_info_name + self.record_file(filename=filename, filecode="bezirk") + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES('bundesland','Bundesland') ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f, delimiter=";") + next(reader) + next(reader) + next(reader) + ans = [r for r in reader] + ans.pop(-1) # last line and 3 first lines are just empty/info + extras.execute_batch( + self.db.cursor, + """INSERT INTO zones(id,name,level) VALUES(%s,%s,(SELECT id FROM zone_levels WHERE name='bundesland')) ON CONFLICT DO NOTHING;""", + ((int(r[0]), r[1]) for r in ans), + ) + self.db.connection.commit() + + def fill_country(self): + self.logger.info("Filling Country") + + self.db.cursor.execute( + """INSERT INTO zone_levels(name,pretty_name) VALUES('country','Country') ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + self.db.cursor.execute( + """INSERT INTO zones(name,code,level) VALUES('Österreich','AT',(SELECT id FROM zone_levels WHERE name='country')) ON CONFLICT DO NOTHING;""" + ) + self.db.connection.commit() + + def fill_parents_zs_g(self): + self.logger.info("Filling zaehlsprengel gemeinde parents") + + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -381,12 +500,14 @@ def fill_parents_zs_g(self): AND zz.id/1000=zg.id ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_parents_zs_bz(self): - self.logger.info('Filling zaehlsprengel bezirk parents') - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_parents_zs_bz(self): + self.logger.info("Filling zaehlsprengel bezirk parents") + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -396,12 +517,14 @@ def fill_parents_zs_bz(self): AND zz.id/100000=zg.id ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_parents_zs_bl(self): - self.logger.info('Filling zaehlsprengel bundesland parents') - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_parents_zs_bl(self): + self.logger.info("Filling zaehlsprengel bundesland parents") + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -411,12 +534,14 @@ def fill_parents_zs_bl(self): AND zz.id/10000000=zg.id ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_parents_g_bz(self): - self.logger.info('Filling gemeinde bezirk parents') - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_parents_g_bz(self): + self.logger.info("Filling gemeinde bezirk parents") + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -426,12 +551,14 @@ def fill_parents_g_bz(self): AND zz.id/100=zg.id ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_parents_g_bl(self): - self.logger.info('Filling gemeinde bundesland parents') - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_parents_g_bl(self): + self.logger.info("Filling gemeinde bundesland parents") + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -441,12 +568,14 @@ def fill_parents_g_bl(self): AND zz.id/10000=zg.id ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_parents_bz_bl(self): - self.logger.info('Filling bezirk bundesland parents') - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_parents_bz_bl(self): + self.logger.info("Filling bezirk bundesland parents") + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -456,12 +585,14 @@ def fill_parents_bz_bl(self): AND zz.id/100=zg.id ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_parents_country(self): - self.logger.info('Filling all country parents') - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_parents_country(self): + self.logger.info("Filling all country parents") + self.db.cursor.execute( + """ INSERT INTO zone_parents(parent_level,parent,child_level,child) (SELECT zg.level,zg.id,zz.level,zz.id FROM zones zg @@ -472,18 +603,20 @@ def fill_parents_country(self): ON zz.level=(SELECT id FROM zone_levels WHERE name=l.lev_name) ) ON CONFLICT DO NOTHING - ;''') - self.db.connection.commit() - - def fill_gis_g(self,gis_type=None): - ''' - Should be executed after fill_gis_zs - ''' - if gis_type is None: - gis_type = self.gis_type - self.logger.info('Filling gemeinde GIS') - - self.db.cursor.execute(''' + ;""" + ) + self.db.connection.commit() + + def fill_gis_g(self, gis_type=None): + """ + Should be executed after fill_gis_zs + """ + if gis_type is None: + gis_type = self.gis_type + self.logger.info("Filling gemeinde GIS") + + self.db.cursor.execute( + """ INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) SELECT zp.parent,zp.parent_level,ST_Union(gd.geom),ST_Centroid(ST_Union(gd.geom)),(SELECT id FROM gis_types WHERE name=%s) FROM gis_data gd @@ -494,19 +627,25 @@ def fill_gis_g(self,gis_type=None): AND zp.child_level=(SELECT id FROM zone_levels WHERE name='zaehlsprengel') GROUP BY zp.parent,zp.parent_level ON CONFLICT DO NOTHING - ;''',(gis_type,gis_type,)) - self.db.connection.commit() - - def fill_gis_bz(self,gis_type=None): - ''' - Should be executed after fill_gis_g - ''' - if gis_type is None: - gis_type = self.gis_type - - self.logger.info('Filling bezirk GIS') - - self.db.cursor.execute(''' + ;""", + ( + gis_type, + gis_type, + ), + ) + self.db.connection.commit() + + def fill_gis_bz(self, gis_type=None): + """ + Should be executed after fill_gis_g + """ + if gis_type is None: + gis_type = self.gis_type + + self.logger.info("Filling bezirk GIS") + + self.db.cursor.execute( + """ INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) SELECT zp.parent,zp.parent_level,ST_Union(gd.geom),ST_Centroid(ST_Union(gd.geom)),(SELECT id FROM gis_types WHERE name=%s) FROM gis_data gd @@ -517,19 +656,25 @@ def fill_gis_bz(self,gis_type=None): AND zp.child_level=(SELECT id FROM zone_levels WHERE name='gemeinde') GROUP BY zp.parent,zp.parent_level ON CONFLICT DO NOTHING - ;''',(gis_type,gis_type,)) - self.db.connection.commit() - - def fill_gis_bl(self,gis_type=None): - ''' - Should be executed after fill_gis_bz - ''' - if gis_type is None: - gis_type = self.gis_type - - self.logger.info('Filling bundesland GIS') - - self.db.cursor.execute(''' + ;""", + ( + gis_type, + gis_type, + ), + ) + self.db.connection.commit() + + def fill_gis_bl(self, gis_type=None): + """ + Should be executed after fill_gis_bz + """ + if gis_type is None: + gis_type = self.gis_type + + self.logger.info("Filling bundesland GIS") + + self.db.cursor.execute( + """ INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) SELECT zp.parent,zp.parent_level,ST_Union(gd.geom),ST_Centroid(ST_Union(gd.geom)),(SELECT id FROM gis_types WHERE name=%s) FROM gis_data gd @@ -540,17 +685,22 @@ def fill_gis_bl(self,gis_type=None): AND zp.child_level=(SELECT id FROM zone_levels WHERE name='bezirk') GROUP BY zp.parent,zp.parent_level ON CONFLICT DO NOTHING - ;''',(gis_type,gis_type,)) - self.db.connection.commit() - - - def fill_gis_country(self,gis_type=None): - if gis_type is None: - gis_type = self.gis_type - - self.logger.info('Filling country GIS') - - self.db.cursor.execute(''' + ;""", + ( + gis_type, + gis_type, + ), + ) + self.db.connection.commit() + + def fill_gis_country(self, gis_type=None): + if gis_type is None: + gis_type = self.gis_type + + self.logger.info("Filling country GIS") + + self.db.cursor.execute( + """ INSERT INTO gis_data(zone_id,zone_level,geom,center,gis_type) SELECT zp.parent,zp.parent_level,ST_Union(gd.geom),ST_Centroid(ST_Union(gd.geom)),(SELECT id FROM gis_types WHERE name=%s) FROM gis_data gd @@ -561,26 +711,33 @@ def fill_gis_country(self,gis_type=None): AND zp.child_level=(SELECT id FROM zone_levels WHERE name='bundesland') GROUP BY zp.parent,zp.parent_level ON CONFLICT DO NOTHING - ;''',(gis_type,gis_type,)) - self.db.connection.commit() + ;""", + ( + gis_type, + gis_type, + ), + ) + self.db.connection.commit() class SimplifiedZSFiller(ZaehlsprengelFiller): - def __init__(self,**kwargs): - ZaehlsprengelFiller.__init__(self,simplified=True,**kwargs) - -class PopulationZSFiller(ZaehlsprengelFiller): - def __init__(self,force=False,**kwargs): - ZaehlsprengelFiller.__init__(self,**kwargs) - self.force = force - self.name = 'population_zs' + def __init__(self, **kwargs): + ZaehlsprengelFiller.__init__(self, simplified=True, **kwargs) - def apply(self): - if self.force or not self.check_done(): - self.fill_population() - def check_done(self): - self.db.cursor.execute(''' +class PopulationZSFiller(ZaehlsprengelFiller): + def __init__(self, force=False, **kwargs): + ZaehlsprengelFiller.__init__(self, **kwargs) + self.force = force + self.name = "population_zs" + + def apply(self): + if self.force or not self.check_done(): + self.fill_population() + + def check_done(self): + self.db.cursor.execute( + """ SELECT z.id, z.level, zat.id--,s.id FROM zones z INNER JOIN zone_attribute_types zat @@ -595,74 +752,82 @@ def check_done(self): ON za.zone=zp.child AND za.zone_level=zp.child_level AND za.attribute=zat.id LIMIT 1 - ;''') - return (self.db.cursor.fetchone() is not None) - + ;""" + ) + return self.db.cursor.fetchone() is not None class PLZFiller(fillers.Filler): - ''' - Filling in PLZ info from statistik.at - ''' - - def __init__(self, - file_info="http://www.statistik.at/verzeichnis/reglisten/gemliste_knz.csv", - file_info_name="gemliste_knz.csv", - force=False, - **kwargs): - self.force = force - self.file_info = file_info - self.file_info_name = file_info_name - fillers.Filler.__init__(self,name='PLZ_gemeinde',**kwargs) - - def prepare(self): - if self.data_folder is None: - self.data_folder = self.db.data_folder - data_folder = self.data_folder - - #create folder if needed - if not os.path.exists(data_folder): - os.makedirs(data_folder) - - # self.db.cursor.execute(''' - # SELECT * FROM plz_gemeinde - # LIMIT 1 - # ;''') - # query_ans = self.db.cursor.fetchone() - query_ans = None - if query_ans is not None: - self.done = True - else: - if not os.path.exists(os.path.join(data_folder,self.file_info_name)): - if not os.path.exists(os.path.join(data_folder,self.file_info_name)): - self.download(url=self.file_info,destination=self.file_info_name) - - def apply(self): - self.fill_plz() - - def fill_plz(self,filename=None): - - self.logger.info('Filling countries') - if filename is None: - filename = self.file_info_name # for children classes - self.record_file(filename=filename,filecode='plz_gemeinde') - with open(os.path.join(self.data_folder,filename),'r') as f: - reader = csv.reader(f,delimiter=';') - next(reader) #remove header - next(reader) #remove header - next(reader) #remove header - ans = [r for r in reader] - ans.pop(-1) - insert_input = [] - for (gd,gd_n,gd2,st,plz,plz_other) in ans: - insert_input.append((gd2,gd_n,plz)) - if plz_other != '': - for plz_o in set(plz_other.split(' ')): - insert_input.append((gd2,gd_n,plz_o)) - extras.execute_batch(self.db.cursor,'''INSERT INTO plz_gemeinde(plz,gemeinde,gemeinde_name) + """ + Filling in PLZ info from statistik.at + """ + + def __init__( + self, + file_info="http://www.statistik.at/verzeichnis/reglisten/gemliste_knz.csv", + file_info_name="gemliste_knz.csv", + force=False, + **kwargs, + ): + self.force = force + self.file_info = file_info + self.file_info_name = file_info_name + fillers.Filler.__init__(self, name="PLZ_gemeinde", **kwargs) + + def prepare(self): + if self.data_folder is None: + self.data_folder = self.db.data_folder + data_folder = self.data_folder + + # create folder if needed + if not os.path.exists(data_folder): + os.makedirs(data_folder) + + # self.db.cursor.execute(''' + # SELECT * FROM plz_gemeinde + # LIMIT 1 + # ;''') + # query_ans = self.db.cursor.fetchone() + query_ans = None + if query_ans is not None: + self.done = True + else: + if not os.path.exists(os.path.join(data_folder, self.file_info_name)): + if not os.path.exists(os.path.join(data_folder, self.file_info_name)): + self.download(url=self.file_info, destination=self.file_info_name) + + def apply(self): + self.fill_plz() + + def fill_plz(self, filename=None): + self.logger.info("Filling countries") + if filename is None: + filename = self.file_info_name # for children classes + self.record_file(filename=filename, filecode="plz_gemeinde") + with open(os.path.join(self.data_folder, filename), "r") as f: + reader = csv.reader(f, delimiter=";") + next(reader) # remove header + next(reader) # remove header + next(reader) # remove header + ans = [r for r in reader] + ans.pop(-1) + insert_input = [] + for gd, gd_n, gd2, st, plz, plz_other in ans: + insert_input.append((gd2, gd_n, plz)) + if plz_other != "": + for plz_o in set(plz_other.split(" ")): + insert_input.append((gd2, gd_n, plz_o)) + extras.execute_batch( + self.db.cursor, + """INSERT INTO plz_gemeinde(plz,gemeinde,gemeinde_name) VALUES(%(plz)s, %(gemeinde)s, %(gemeinde_name)s) ON CONFLICT DO NOTHING - ;''',({'plz':plz,'gemeinde':gd,'gemeinde_name':gd_n} for (gd,gd_n,plz) in insert_input)) - self.db.connection.commit() + ;""", + ( + {"plz": plz, "gemeinde": gd, "gemeinde_name": gd_n} + for (gd, gd_n, plz) in insert_input + ), + ) + self.db.connection.commit() diff --git a/gis_fillers/getters/zone_getters.py b/gis_fillers/getters/zone_getters.py index 0c1e208..6d89244 100644 --- a/gis_fillers/getters/zone_getters.py +++ b/gis_fillers/getters/zone_getters.py @@ -8,22 +8,31 @@ import geopandas as gpd import pandas as pd + class PopulationGetter(Getter): - ''' - Returns a geopandas dataframe with population per defined area level - ''' - columns = ('Zone','ZoneID','population','geometry','area') - def __init__(self,zone_level='bezirk',zone_attribute='population',simplified=True,**kwargs): - Getter.__init__(self,**kwargs) - self.zone_level = zone_level - self.zone_attribute = zone_attribute - if simplified: - self.target_gt = 'zaehlsprengel_simplified' - else: - self.target_gt = 'zaehlsprengel' + """ + Returns a geopandas dataframe with population per defined area level + """ + + columns = ("Zone", "ZoneID", "population", "geometry", "area") + + def __init__( + self, + zone_level="bezirk", + zone_attribute="population", + simplified=True, + **kwargs + ): + Getter.__init__(self, **kwargs) + self.zone_level = zone_level + self.zone_attribute = zone_attribute + if simplified: + self.target_gt = "zaehlsprengel_simplified" + else: + self.target_gt = "zaehlsprengel" - def query(self): - return ''' + def query(self): + return """ SELECT q1.id,q1.level,q2.population,q1.name,q1.geometry, q1.area FROM (SELECT z.id,z.level,z.name, ST_AsText(gd.geom) AS geometry, ST_Area(gd.geom,false)/10^6 AS area FROM zones z @@ -59,34 +68,53 @@ def query(self): GROUP BY z.id,z.level,z.name ) AS q2 ON q1.id=q2.id AND q1.level=q2.level - ;''' - - - def query_attributes(self): - return { - 'zone_level':self.zone_level, - 'zone_attribute':self.zone_attribute, - 'target_gt':self.target_gt - } + ;""" - def parse_results(self,query_result): - return [{'Zone':bez,'ZoneID':zid,'population':0 if np.isnan(pop) else int(pop),'geometry':shapely.wkt.loads(geo),'area':area} for (zid,zlvl,pop,bez,geo,area) in query_result] + def query_attributes(self): + return { + "zone_level": self.zone_level, + "zone_attribute": self.zone_attribute, + "target_gt": self.target_gt, + } - def get(self,db,**kwargs): - db.cursor.execute(self.query(),self.query_attributes()) - query_result = list(db.cursor.fetchall()) - gdf = gpd.GeoDataFrame(self.parse_results(query_result=query_result),crs='epsg:4326',columns=self.columns) - return gdf + def parse_results(self, query_result): + return [ + { + "Zone": bez, + "ZoneID": zid, + "population": 0 if np.isnan(pop) else int(pop), + "geometry": shapely.wkt.loads(geo), + "area": area, + } + for (zid, zlvl, pop, bez, geo, area) in query_result + ] + def get(self, db, **kwargs): + db.cursor.execute(self.query(), self.query_attributes()) + query_result = list(db.cursor.fetchall()) + gdf = gpd.GeoDataFrame( + self.parse_results(query_result=query_result), + crs="epsg:4326", + columns=self.columns, + ) + return gdf class PopulationDensityGetter(PopulationGetter): - ''' - Returns a geopandas dataframe with population density in people/km2 per defined area level - ''' + """ + Returns a geopandas dataframe with population density in people/km2 per defined area level + """ - columns = ('Zone','ZoneID','population_density','geometry','area') - - def parse_results(self,query_result): - return [{'Zone':bez,'ZoneID':zid,'population_density':0 if np.isnan(pop) else int(pop)/area,'geometry':shapely.wkt.loads(geo),'area':area} for (zid,zlvlv,pop,bez,geo,area) in query_result] + columns = ("Zone", "ZoneID", "population_density", "geometry", "area") + def parse_results(self, query_result): + return [ + { + "Zone": bez, + "ZoneID": zid, + "population_density": 0 if np.isnan(pop) else int(pop) / area, + "geometry": shapely.wkt.loads(geo), + "area": area, + } + for (zid, zlvlv, pop, bez, geo, area) in query_result + ] diff --git a/scripts/example_script.py b/scripts/example_script.py index 53d988e..f6beca2 100644 --- a/scripts/example_script.py +++ b/scripts/example_script.py @@ -10,11 +10,11 @@ from matplotlib import pyplot as plt conninfo = { - 'host':'195.154.70.113', - 'port':64741, - 'database':'example_db', # CHANGE TO 'playground' to have write rights - 'user':'wschuell', # CHANGE TO YOUR USERNAME - 'data_folder': os.path.join(os.path.dirname(__file__),'data_folder') + "host": "195.154.70.113", + "port": 64741, + "database": "example_db", # CHANGE TO 'playground' to have write rights + "user": "wschuell", # CHANGE TO YOUR USERNAME + "data_folder": os.path.join(os.path.dirname(__file__), "data_folder"), } # conninfo = { @@ -31,17 +31,17 @@ ######## Getter wrappers around complex queries (see the corresponding imported files) -zone_level = 'bezirk' -gdf = zone_getters.PopulationGetter(db=db,zone_level=zone_level).get_result() # gets a geopandas dataframe with various info -- super fast because SQL query behind +zone_level = "bezirk" +gdf = zone_getters.PopulationGetter( + db=db, zone_level=zone_level +).get_result() # gets a geopandas dataframe with various info -- super fast because SQL query behind print(gdf) -gdf.plot(column='population', legend=True) -plt.title('Population at {} level'.format(zone_level)) +gdf.plot(column="population", legend=True) +plt.title("Population at {} level".format(zone_level)) plt.show() - - # zone_level = 'zaehlsprengel' # gdf = zone_getters.PopulationDensityGetter(db=db,zone_level=zone_level).get_result() # # area for density is from the exact definition of the zone geometries, but display uses the simplified ones. Can especially impact the values for zaehlsprengel level in Vienna @@ -55,57 +55,68 @@ ## Extra: using hexagons H3 __hexagons_ # needs to be filled beforehand, see filldb script -zone_level = 'zaehlsprengel' -gdf = zone_getters.PopulationDensityGetter(db=db,zone_level=zone_level,simplified=True).get_result() # gets a geopandas dataframe with various info -- super fast because SQL query behind +zone_level = "zaehlsprengel" +gdf = zone_getters.PopulationDensityGetter( + db=db, zone_level=zone_level, simplified=True +).get_result() # gets a geopandas dataframe with various info -- super fast because SQL query behind print(gdf) -gdf.plot(column='population_density', legend=True) -plt.title('Population at {} level'.format(zone_level)) +gdf.plot(column="population_density", legend=True) +plt.title("Population at {} level".format(zone_level)) plt.show() -zone_level = 'bezirk_922_hexagons_8' +zone_level = "bezirk_922_hexagons_8" # zone_level = 'bezirk_922_hexagons_9' -gdf = zone_getters.PopulationGetter(db=db,zone_level=zone_level,simplified=False).get_result() # gets a geopandas dataframe with various info -- super fast because SQL query behind +gdf = zone_getters.PopulationGetter( + db=db, zone_level=zone_level, simplified=False +).get_result() # gets a geopandas dataframe with various info -- super fast because SQL query behind print(gdf) -gdf.plot(column='population', legend=True) -plt.title('Population at {} level'.format(zone_level)) +gdf.plot(column="population", legend=True) +plt.title("Population at {} level".format(zone_level)) plt.show() -zone_level = 'country_AT_hexagons_4' +zone_level = "country_AT_hexagons_4" # zone_level = 'country_AT_hexagons_5' -gdf = zone_getters.PopulationDensityGetter(db=db,zone_level=zone_level,simplified=False).get_result() +gdf = zone_getters.PopulationDensityGetter( + db=db, zone_level=zone_level, simplified=False +).get_result() # area for density is from the exact definition of the zone geometries, but display uses the simplified ones. Can especially impact the values for zaehlsprengel level in Vienna print(gdf) -gdf.plot(column='population_density',legend=True) -plt.title('Population density at {} level'.format(zone_level)) +gdf.plot(column="population_density", legend=True) +plt.title("Population density at {} level".format(zone_level)) plt.show() ######### Raw queries example: with the countries queries can still take a long time given the detail level (original high precision) -for title,query in [ - ('100 first zones in the DB (random)','''SELECT geom FROM gis_data LIMIT 100;'''), - - ('20 first countries','''SELECT gd.geom FROM zone_levels zl +for title, query in [ + ("100 first zones in the DB (random)", """SELECT geom FROM gis_data LIMIT 100;"""), + ( + "20 first countries", + """SELECT gd.geom FROM zone_levels zl INNER JOIN gis_types gt ON gt.name='zaehlsprengel' AND zl.name='country' INNER JOIN gis_data gd ON gd.gis_type=gt.id AND gd.zone_level=zl.id LIMIT 20 - ;'''), - - ('all countries','''SELECT gd.geom FROM zone_levels zl + ;""", + ), + ( + "all countries", + """SELECT gd.geom FROM zone_levels zl INNER JOIN gis_types gt ON gt.name='zaehlsprengel' AND zl.name='country' INNER JOIN gis_data gd ON gd.gis_type=gt.id AND gd.zone_level=zl.id - ;'''), - - ('40 countries closest to Austria by center',''' + ;""", + ), + ( + "40 countries closest to Austria by center", + """ WITH at_gd AS (SELECT gd.center FROM zones z INNER JOIN zone_levels zl ON z.code ='AT' @@ -123,9 +134,11 @@ ON gd.gis_type=gt.id AND gd.zone_level=zl.id ORDER BY ST_Distance((SELECT center FROM at_gd),gd.center) ASC LIMIT 40 - ;'''), - - ('40 countries closest to Austria by geometry',''' + ;""", + ), + ( + "40 countries closest to Austria by geometry", + """ WITH at_gd AS (SELECT gd.geom FROM zones z INNER JOIN zone_levels zl ON z.code ='AT' @@ -143,11 +156,11 @@ ON gd.gis_type=gt.id AND gd.zone_level=zl.id ORDER BY ST_Distance((SELECT geom FROM at_gd),gd.geom) ASC LIMIT 40 - ;'''), - ]: - - gdf = gpd.GeoDataFrame.from_postgis(sql=query,con=db.connection) - - gdf.plot() - plt.title(title) - plt.show() + ;""", + ), +]: + gdf = gpd.GeoDataFrame.from_postgis(sql=query, con=db.connection) + + gdf.plot() + plt.title(title) + plt.show() diff --git a/scripts/filldb_script.py b/scripts/filldb_script.py index a4145e1..e7a6ed7 100644 --- a/scripts/filldb_script.py +++ b/scripts/filldb_script.py @@ -10,11 +10,11 @@ from matplotlib import pyplot as plt conninfo = { - 'host':'195.154.70.113', - 'port':64741, - 'database':'example_db', # CHANGE TO 'playground' to have write rights - 'user':'wschuell', # CHANGE TO YOUR USERNAME - 'data_folder': os.path.join(os.path.dirname(__file__),'data_folder') + "host": "195.154.70.113", + "port": 64741, + "database": "example_db", # CHANGE TO 'playground' to have write rights + "user": "wschuell", # CHANGE TO YOUR USERNAME + "data_folder": os.path.join(os.path.dirname(__file__), "data_folder"), } # conninfo = { @@ -33,17 +33,22 @@ # db.clean_db() # Erases all data -db.init_db() # Creates the structure +db.init_db() # Creates the structure -db.add_filler(zones.zaehlsprengel.ZaehlsprengelFiller()) # Fills in data for AT -db.add_filler(zones.zaehlsprengel.SimplifiedZSFiller()) # Fills in data for AT but with lower precision for geometries (uses mapshaper - to be installed separately) -db.add_filler(zones.zaehlsprengel.PLZFiller()) # Fills in zip code data for AT +db.add_filler(zones.zaehlsprengel.ZaehlsprengelFiller()) # Fills in data for AT +db.add_filler( + zones.zaehlsprengel.SimplifiedZSFiller() +) # Fills in data for AT but with lower precision for geometries (uses mapshaper - to be installed separately) +db.add_filler(zones.zaehlsprengel.PLZFiller()) # Fills in zip code data for AT db.add_filler(zones.countries.CountriesFiller()) -db.add_filler(zones.hexagons.HexagonsFiller(res=8,target_zone=922,target_zone_level='bezirk')) +db.add_filler( + zones.hexagons.HexagonsFiller(res=8, target_zone=922, target_zone_level="bezirk") +) # db.add_filler(zones.hexagons.HexagonsFiller(res=9,target_zone=922,target_zone_level='bezirk')) -db.add_filler(zones.hexagons.HexagonsFiller(res=4,target_zone='AT',target_zone_level='country')) +db.add_filler( + zones.hexagons.HexagonsFiller(res=4, target_zone="AT", target_zone_level="country") +) # db.add_filler(zones.hexagons.HexagonsFiller(res=5,target_zone='AT',target_zone_level='country')) db.fill_db() - diff --git a/setup.py b/setup.py index bd69468..b03ecbd 100644 --- a/setup.py +++ b/setup.py @@ -7,22 +7,24 @@ def version(): - with open('gis_fillers/_version.py') as f: + with open("gis_fillers/_version.py") as f: return re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", f.read()).group(1) + def requirements(): - with open('requirements.txt') as f: - return f.readlines() - -setup(name='gis_fillers', - version=version(), - packages=['gis_fillers'],#find_packages(), - install_requires=[requirements()], - author='William Schueller', - author_email='', - description='', - url='', - license='GNU AFFERO GENERAL PUBLIC LICENSE Version 3', - include_package_data=True, - ) - + with open("requirements.txt") as f: + return f.readlines() + + +setup( + name="gis_fillers", + version=version(), + packages=["gis_fillers"], # find_packages(), + install_requires=[requirements()], + author="William Schueller", + author_email="", + description="", + url="", + license="GNU AFFERO GENERAL PUBLIC LICENSE Version 3", + include_package_data=True, +) diff --git a/tests/testmodule/test_basic.py b/tests/testmodule/test_basic.py index a2c08eb..7f0ddf6 100644 --- a/tests/testmodule/test_basic.py +++ b/tests/testmodule/test_basic.py @@ -1,14 +1,15 @@ import os import importlib + def test_basic(): - assert True + assert True -def test_import(): - path = os.path.abspath(__file__) - dir_name = os.path.dirname(os.path.dirname(os.path.dirname(path))) - libname = os.path.basename(dir_name) - if libname == 'pylib_template': - libname = 'PYLIB' - importlib.import_module(libname) +def test_import(): + path = os.path.abspath(__file__) + dir_name = os.path.dirname(os.path.dirname(os.path.dirname(path))) + libname = os.path.basename(dir_name) + if libname == "pylib_template": + libname = "PYLIB" + importlib.import_module(libname) diff --git a/tests/testmodule/test_db.py b/tests/testmodule/test_db.py index 9b621a9..9e68743 100644 --- a/tests/testmodule/test_db.py +++ b/tests/testmodule/test_db.py @@ -8,47 +8,58 @@ from gis_fillers.getters import zone_getters conninfo = { - 'host':'localhost', - 'port':5432, - 'database':'test_gis_fillers', - 'user':'postgres', - 'data_folder': os.path.join(os.path.dirname(os.path.dirname(__file__)),'data_folder') + "host": "localhost", + "port": 5432, + "database": "test_gis_fillers", + "user": "postgres", + "data_folder": os.path.join( + os.path.dirname(os.path.dirname(__file__)), "data_folder" + ), } def test_connect(): - db = Database(**conninfo) + db = Database(**conninfo) + def test_init(): - db = Database(**conninfo) - db.init_db() + db = Database(**conninfo) + db.init_db() + def test_clean(): - db = Database(**conninfo) - db.clean_db() - db.init_db() + db = Database(**conninfo) + db.clean_db() + db.init_db() + @pytest.fixture def maindb(): - db = Database(**conninfo) - db.init_db() - yield db - db.connection.close() + db = Database(**conninfo) + db.init_db() + yield db + db.connection.close() def test_countries(maindb): - maindb.add_filler(zones.countries.CountriesFiller()) - maindb.fill_db() + maindb.add_filler(zones.countries.CountriesFiller()) + maindb.fill_db() def test_zs(maindb): - maindb.add_filler(zones.zaehlsprengel.ZaehlsprengelFiller()) - maindb.fill_db() + maindb.add_filler(zones.zaehlsprengel.ZaehlsprengelFiller()) + maindb.fill_db() def test_simplified_zs(maindb): - maindb.add_filler(zones.zaehlsprengel.SimplifiedZSFiller(simplify_engine='topojson',geojson_gis_info_name="topojson_{YEAR}0101.geojson",)) - maindb.fill_db() + maindb.add_filler( + zones.zaehlsprengel.SimplifiedZSFiller( + simplify_engine="topojson", + geojson_gis_info_name="topojson_{YEAR}0101.geojson", + ) + ) + maindb.fill_db() + # def test_simplified_zs_mapshaper(maindb): # maindb.add_filler(zones.zaehlsprengel.SimplifiedZSFiller(simplify_engine='mapshaper',geojson_gis_info_name="mapshaper_{YEAR}0101.geojson",)) @@ -56,53 +67,70 @@ def test_simplified_zs(maindb): def test_plz(maindb): - maindb.add_filler(zones.zaehlsprengel.PLZFiller()) - maindb.fill_db() - + maindb.add_filler(zones.zaehlsprengel.PLZFiller()) + maindb.fill_db() res_list = [ - 3, - 4, - ] + 3, + 4, +] + + @pytest.fixture(params=res_list) def res(request): - return request.param + return request.param + + +country_list = ["AT", "FR"] + -country_list = [ - 'AT', - 'FR' - ] @pytest.fixture(params=country_list) def country(request): - return request.param + return request.param + + +def test_hexagons(maindb, res, country): + maindb.add_filler( + zones.hexagons.HexagonsFiller( + res=res, target_zone=country, target_zone_level="country" + ) + ) + maindb.fill_db() + -def test_hexagons(maindb,res,country): - maindb.add_filler(zones.hexagons.HexagonsFiller(res=res,target_zone=country,target_zone_level='country')) - maindb.fill_db() +bezirk_list = [918, 922] -bezirk_list = [ - 918, - 922 - ] @pytest.fixture(params=bezirk_list) def bezirk(request): - return request.param + return request.param + res_bezirk_list = [ - 7, - 8, - ] + 7, + 8, +] + + @pytest.fixture(params=res_bezirk_list) def res_bezirk(request): - return request.param + return request.param + -def test_hexagons_bezirk(maindb,res_bezirk,bezirk): - maindb.add_filler(zones.hexagons.HexagonsFiller(res=res_bezirk,target_zone=bezirk,target_zone_level='bezirk')) - maindb.fill_db() +def test_hexagons_bezirk(maindb, res_bezirk, bezirk): + maindb.add_filler( + zones.hexagons.HexagonsFiller( + res=res_bezirk, target_zone=bezirk, target_zone_level="bezirk" + ) + ) + maindb.fill_db() def test_getters(maindb): - zone_getters.PopulationGetter(db=maindb,zone_level='bezirk',simplified=False).get_result() - zone_getters.PopulationDensityGetter(db=maindb,zone_level='bezirk',simplified=False).get_result() + zone_getters.PopulationGetter( + db=maindb, zone_level="bezirk", simplified=False + ).get_result() + zone_getters.PopulationDensityGetter( + db=maindb, zone_level="bezirk", simplified=False + ).get_result()