diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 0e344f516..174a75503 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -43,6 +43,9 @@ Bug fixes different CRS. If they do, all files are converted to a single, common CRS to make sure they can be merged without issues (:pull:`1698`). By `Patrick Schmitt `_ +- COPDEM data is now downloaded from public http server (no credentials + required) (:pull:`1684`). + By `Codruț Diaconu `_ v1.6.1 (August 27, 2023) diff --git a/oggm/tests/test_utils.py b/oggm/tests/test_utils.py index 151f016c2..fb161bd1e 100644 --- a/oggm/tests/test_utils.py +++ b/oggm/tests/test_utils.py @@ -2230,8 +2230,7 @@ def test_copdem90(self): # Make a fake topo file deep_path = os.path.join(self.dldir, - 'DEM1_SAR_DGE_90_20110517T170701_20140817T170857' - '_ADS_000000_3682.DEM', + 'COP-DEM_GLO-90-DGED__2023_1', 'Copernicus_DSM_30_N46_00_E010_00', 'DEM') utils.mkdir(deep_path) upper_path = os.path.dirname(os.path.dirname(deep_path)) @@ -2242,14 +2241,13 @@ def test_copdem90(self): archive='tar', extension='.tar') def down_check(url, *args, **kwargs): - expected = ('ftps://cdsdata.copernicus.eu:990/datasets/' - 'COP-DEM_GLO-90-DGED/2022_1/' - 'DEM1_SAR_DGE_90_20110517T170701_20140817T170857_ADS_' - '000000_3682.DEM.tar') + expected = ('https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload/' + 'COP-DEM_GLO-90-DGED__2023_1/' + 'Copernicus_DSM_30_N46_00_E010_00.tar') self.assertEqual(expected, url) return tf - with FakeDownloadManager('download_with_authentication', down_check): + with FakeDownloadManager('file_downloader', down_check): of, source = utils.get_topo_file([10.5, 10.8], [46.6, 46.8], source='COPDEM90') @@ -2260,8 +2258,7 @@ def test_copdem30(self): # Make a fake topo file deep_path = os.path.join(self.dldir, - 'DEM1_SAR_DGE_30_20110517T170701_20140817T170857' - '_ADS_000000_bma2.DEM', + 'COP-DEM_GLO-30-DGED__2023_1', 'Copernicus_DSM_10_N46_00_E010_00', 'DEM') utils.mkdir(deep_path) upper_path = os.path.dirname(os.path.dirname(deep_path)) @@ -2272,14 +2269,13 @@ def test_copdem30(self): archive='tar', extension='.tar') def down_check(url, *args, **kwargs): - expected = ('ftps://cdsdata.copernicus.eu:990/datasets/' - 'COP-DEM_GLO-30-DGED/2022_1/' - 'DEM1_SAR_DGE_30_20110517T170701_20140817T170857_ADS_' - '000000_bma2.DEM.tar') + expected = ('https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload/' + 'COP-DEM_GLO-30-DGED__2023_1/' + 'Copernicus_DSM_10_N46_00_E010_00.tar') self.assertEqual(expected, url) return tf - with FakeDownloadManager('download_with_authentication', down_check): + with FakeDownloadManager('file_downloader', down_check): of, source = utils.get_topo_file([10.5, 10.8], [46.6, 46.8], source='COPDEM30') @@ -2762,29 +2758,25 @@ def test_dem3_viewpano_zone(self): def test_copdemzone(self): z = utils.copdem_zone([-77.6, -77.3], [-9.8, -9.5], 'COPDEM90') self.assertTrue(len(z) == 1) - self.assertEqual(('DEM1_SAR_DGE_90_20110427T104941_20140819T105300_' - 'ADS_000000_7913.DEM.tar'), z[0][0]) - self.assertEqual('Copernicus_DSM_30_S10_00_W078_00', z[0][1]) + self.assertEqual('Copernicus_DSM_30_S10_00_W078_00', z[0]) z = utils.copdem_zone([-77.6, -77.3], [-9.8, -9.5], 'COPDEM30') self.assertTrue(len(z) == 1) - self.assertEqual(('DEM1_SAR_DGE_30_20110427T104941_20140819T105300_' - 'ADS_000000_elyt.DEM.tar'), z[0][0]) - self.assertEqual('Copernicus_DSM_10_S10_00_W078_00', z[0][1]) + self.assertEqual('Copernicus_DSM_10_S10_00_W078_00', z[0]) z = utils.copdem_zone([7.89, 8.12], [46.37, 46.59], 'COPDEM90') self.assertTrue(len(z) == 2) self.assertTrue('Copernicus_DSM_30_N46_00_E008_00' in - [z[0][1], z[1][1]]) + [z[0], z[1]]) self.assertTrue('Copernicus_DSM_30_N46_00_E007_00' in - [z[0][1], z[1][1]]) + [z[0], z[1]]) z = utils.copdem_zone([7.89, 8.12], [46.37, 46.59], 'COPDEM30') self.assertTrue(len(z) == 2) self.assertTrue('Copernicus_DSM_10_N46_00_E008_00' in - [z[0][1], z[1][1]]) + [z[0], z[1]]) self.assertTrue('Copernicus_DSM_10_N46_00_E007_00' in - [z[0][1], z[1][1]]) + [z[0], z[1]]) def test_is_dem_source_available(self): assert utils.is_dem_source_available('SRTM', [11, 11], [47, 47]) diff --git a/oggm/utils/_downloads.py b/oggm/utils/_downloads.py index 6152b2101..1ac339c44 100644 --- a/oggm/utils/_downloads.py +++ b/oggm/utils/_downloads.py @@ -1137,12 +1137,12 @@ def _download_topo_file_from_cluster_unlocked(fname): return outpath -def _download_copdem_file(cppfile, tilename, source): +def _download_copdem_file(tilename, source): with get_lock(): - return _download_copdem_file_unlocked(cppfile, tilename, source) + return _download_copdem_file_unlocked(tilename, source) -def _download_copdem_file_unlocked(cppfile, tilename, source): +def _download_copdem_file_unlocked(tilename, source): """Checks if Copernicus DEM file is in the directory, if not download it. cppfile : name of the tarfile to download @@ -1164,12 +1164,9 @@ def _download_copdem_file_unlocked(cppfile, tilename, source): return demfile # Did we download it yet? - ftpfile = ('ftps://cdsdata.copernicus.eu:990/' + - 'datasets/COP-DEM_GLO-{}-DGED/2022_1/'.format(source[-2:]) + - cppfile) - - dest_file = download_with_authentication(ftpfile, - 'spacedata.copernicus.eu') + url = (f"https://prism-dem-open.copernicus.eu/pd-desk-open-access/prismDownload" + f"/COP-DEM_GLO-{source[-2:]}-DGED__2023_1/{tilename}.tar") + dest_file = file_downloader(url) # None means we tried hard but we couldn't find it if not dest_file: @@ -1563,7 +1560,21 @@ def alaska_dem_zone(lon_ex, lat_ex): def copdem_zone(lon_ex, lat_ex, source): - """Returns a list of Copernicus DEM tarfile and tilename tuples + """Returns a list of Copernicus DEM tilenames. + + New: + We now go for the PRISM server download, which has a slightly different + API. + + Parse available datasets: + curl -k -H "accept: csv" https://prism-dem-open.copernicus.eu/pd-desk-open-access/publicDemURLs + + Parse available tiles: + curl -k -H "accept: csv" https://prism-dem-open.copernicus.eu/pd-desk-open-access/publicDemURLs/COP-DEM_GLO-30-DGED__2023_1 + + But I'm not sure why parsing the tiles is necessary, we can just use the + tilename as is and try to download it. If it doesn't exist, it's probably + OK to skip it. """ # because we use both meters and arc secs in our filenames... @@ -1574,13 +1585,6 @@ def copdem_zone(lon_ex, lat_ex, source): else: raise InvalidDEMError('COPDEM Version not valid.') - # either reuse or load lookup table - if source in cfg.DATA: - df = cfg.DATA[source] - else: - df = pd.read_csv(get_demo_file('{}_2022_1.csv'.format(source.lower()))) - cfg.DATA[source] = df - # adding small buffer for unlikely case where one lon/lat_ex == xx.0 lons = np.arange(np.floor(lon_ex[0]-1e-9), np.ceil(lon_ex[1]+1e-9)) lats = np.arange(np.floor(lat_ex[0]-1e-9), np.ceil(lat_ex[1]+1e-9)) @@ -1594,14 +1598,7 @@ def copdem_zone(lon_ex, lat_ex, source): ew = 'W' if lon < 0 else 'E' lat_str = '{}{:02.0f}'.format(ns, abs(lat)) lon_str = '{}{:03.0f}'.format(ew, abs(lon)) - try: - filename = df.loc[(df['Long'] == lon_str) & - (df['Lat'] == lat_str)]['CPP filename'].iloc[0] - flist.append((filename, - 'Copernicus_DSM_{}_{}_00_{}_00'.format(asec, lat_str, lon_str))) - except IndexError: - # COPDEM is global, if we miss tiles it is probably in the ocean - pass + flist.append('Copernicus_DSM_{}_{}_00_{}_00'.format(asec, lat_str, lon_str)) return flist @@ -2480,9 +2477,9 @@ def get_topo_file(lon_ex=None, lat_ex=None, gdir=None, *, files.append(_download_srtm_file(z)) if source in ['COPDEM30', 'COPDEM90']: - filetuple = copdem_zone(lon_ex, lat_ex, source) - for cpp, eop in filetuple: - files.append(_download_copdem_file(cpp, eop, source)) + tilenames = copdem_zone(lon_ex, lat_ex, source) + for tilename in tilenames: + files.append(_download_copdem_file(tilename, source)) if source == 'NASADEM': zones = nasadem_zone(lon_ex, lat_ex)