From a551aab2ce64a88bbd351c5970dad3ff3c6ee76e Mon Sep 17 00:00:00 2001 From: Adrien Perrin Date: Wed, 10 Jan 2024 09:30:03 +0000 Subject: [PATCH] generalize GPortal AMSR2 normalizer --- metanorm/normalizers/geospaas/gportal_gcom.py | 29 +++++++++------ tests/normalizers/test_gportal_gcom.py | 36 +++++++++++-------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/metanorm/normalizers/geospaas/gportal_gcom.py b/metanorm/normalizers/geospaas/gportal_gcom.py index 981edf4e..7f718b3a 100644 --- a/metanorm/normalizers/geospaas/gportal_gcom.py +++ b/metanorm/normalizers/geospaas/gportal_gcom.py @@ -7,28 +7,29 @@ from .base import GeoSPaaSMetadataNormalizer -class GPortalGCOMAMSR2L3MetadataNormalizer(GeoSPaaSMetadataNormalizer): +class GPortalGCOMWAMSR2MetadataNormalizer(GeoSPaaSMetadataNormalizer): """Generate the properties of a GeoSPaaS Dataset for a GCOM-W AMSR2 - L3 dataset + dataset """ def check(self, raw_metadata): """Checks that the URL starts with the right prefix""" return raw_metadata.get('url', '').startswith( - 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST') + 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/') def get_entry_title(self, raw_metadata): - return 'AMSR2-L3 Sea Surface Temperature' + return 'GCOM-W AMSR2' @utils.raises((AttributeError, KeyError)) def get_entry_id(self, raw_metadata): return utils.NC_H5_FILENAME_MATCHER.search(raw_metadata['url']).group(1) def get_summary(self, raw_metadata): - return utils.dict_to_string({ - utils.SUMMARY_FIELDS['description']: 'GCOM-W AMSR2 data', - utils.SUMMARY_FIELDS['processing_level']: '3' - }) + result = {utils.SUMMARY_FIELDS['description']: 'GCOM-W AMSR2 data'} + processing_level_match = re.match(r'^.*/L([1-3][A-Z]?)(\.[^/]+)?/.*$', raw_metadata['url']) + if processing_level_match: + result[utils.SUMMARY_FIELDS['processing_level']] = processing_level_match.group(1) + return utils.dict_to_string(result) time_patterns = ( ( @@ -41,6 +42,15 @@ def get_summary(self, raw_metadata): utils.create_datetime, lambda time: (time, time + relativedelta(months=1)) ), + ( + re.compile(r'/[A-Z\d]+_' + + utils.YEARMONTHDAY_REGEX + + r'(?P\d{2})' + + r'(?P\d{2})' + + r'_.*\.h5$'), + utils.create_datetime, + lambda time: (time, time + relativedelta(minutes=50)) + ), ) @utils.raises(KeyError) @@ -62,6 +72,3 @@ def get_location_geometry(self, raw_metadata): def get_provider(self, raw_metadata): return utils.get_gcmd_provider(['JP/JAXA/EOC']) - - def get_dataset_parameters(self, raw_metadata): - return utils.create_parameter_list(('sea_surface_temperature',)) diff --git a/tests/normalizers/test_gportal_gcom.py b/tests/normalizers/test_gportal_gcom.py index 0afebd86..3a5316f9 100644 --- a/tests/normalizers/test_gportal_gcom.py +++ b/tests/normalizers/test_gportal_gcom.py @@ -8,10 +8,10 @@ from metanorm.errors import MetadataNormalizationError -class GPortalGCOMAMSR2L3MetadataNormalizerTestCase(unittest.TestCase): +class GPortalGCOMWAMSR2MetadataNormalizerTestCase(unittest.TestCase): """Tests for the GPortal GCOM-W ftp normalizer""" def setUp(self): - self.normalizer = normalizers.geospaas.GPortalGCOMAMSR2L3MetadataNormalizer() + self.normalizer = normalizers.geospaas.GPortalGCOMWAMSR2MetadataNormalizer() def test_check(self): """Test the checking condition""" @@ -23,16 +23,20 @@ def test_check(self): 'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST_25/3/2018/07/' 'GW1AM2_20180703_01D_EQOA_L3SGSSTHB3300300.h5' })) + self.assertTrue(self.normalizer.check({ + 'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L1R/2/2020/02/' + 'GW1AM2_202002011046_045A_L1SGRTBR_2220220.h5' + })) self.assertFalse(self.normalizer.check({})) self.assertFalse(self.normalizer.check({'url': 'ftp://foo/bar'})) def test_entry_title(self): - """entry_title from GPortalGCOMAMSR2L3MetadataNormalizer """ - self.assertEqual(self.normalizer.get_entry_title({}), 'AMSR2-L3 Sea Surface Temperature') + """entry_title from GPortalGCOMWAMSR2MetadataNormalizer """ + self.assertEqual(self.normalizer.get_entry_title({}), 'GCOM-W AMSR2') def test_entry_id(self): - """entry_id from GPortalGCOMAMSR2L3MetadataNormalizer """ + """entry_id from GPortalGCOMWAMSR2MetadataNormalizer """ attributes = { 'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST_25/3/2012/07/' 'GW1AM2_201207031905_134D_L2SGSSTLB3300300.h5'} @@ -49,10 +53,19 @@ def test_entry_id_error(self): self.normalizer.get_entry_id({}) def test_summary(self): - """summary from GPortalGCOMAMSR2L3MetadataNormalizer """ + """summary from GPortalGCOMWAMSR2MetadataNormalizer """ self.assertEqual( - self.normalizer.get_summary({}), + self.normalizer.get_summary({ + 'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST_25/3/2017/12/' + 'GW1AM2_20171201_01D_EQOA_L3SGSSTLB3300300.h5' + }), 'Description: GCOM-W AMSR2 data;Processing level: 3') + self.assertEqual( + self.normalizer.get_summary({ + 'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L1R/2/2020/02/' + 'GW1AM2_202002011046_045A_L1SGRTBR_2220220.h5' + }), + 'Description: GCOM-W AMSR2 data;Processing level: 1R') def test_time_coverage_start_day(self): """Test getting time_coverage_start from a day file""" @@ -118,14 +131,7 @@ def test_gcmd_provider(self): mock_get_gcmd_method.return_value) def test_location_geometry(self): - """geometry from GPortalGCOMAMSR2L3MetadataNormalizer """ + """geometry from GPortalGCOMWAMSR2MetadataNormalizer """ self.assertEqual( self.normalizer.get_location_geometry({}), 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))') - - def test_dataset_parameters(self): - """dataset_parameters from GPortalGCOMAMSR2L3MetadataNormalizer """ - with mock.patch('metanorm.utils.create_parameter_list') as mock_get_gcmd_method: - self.assertEqual( - self.normalizer.get_dataset_parameters({}), - mock_get_gcmd_method.return_value)