Skip to content

Commit

Permalink
Merge pull request #139 from nansencenter/update_amsr2_normalizer
Browse files Browse the repository at this point in the history 
Generalize GPortal AMSR2 normalizer
  • Loading branch information
@aperrin66
aperrin66 authored Jan 10, 2024
2 parents d3b0f61 + a551aab commit 3c8ea53
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 26 deletions.
29 changes: 18 additions & 11 deletions metanorm/normalizers/geospaas/gportal_gcom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,29 @@
from .base import GeoSPaaSMetadataNormalizer


class GPortalGCOMAMSR2L3MetadataNormalizer(GeoSPaaSMetadataNormalizer):
class GPortalGCOMWAMSR2MetadataNormalizer(GeoSPaaSMetadataNormalizer):
"""Generate the properties of a GeoSPaaS Dataset for a GCOM-W AMSR2
L3 dataset
dataset
"""

def check(self, raw_metadata):
"""Checks that the URL starts with the right prefix"""
return raw_metadata.get('url', '').startswith(
'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST')
'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/')

def get_entry_title(self, raw_metadata):
return 'AMSR2-L3 Sea Surface Temperature'
return 'GCOM-W AMSR2'

@utils.raises((AttributeError, KeyError))
def get_entry_id(self, raw_metadata):
return utils.NC_H5_FILENAME_MATCHER.search(raw_metadata['url']).group(1)

def get_summary(self, raw_metadata):
return utils.dict_to_string({
utils.SUMMARY_FIELDS['description']: 'GCOM-W AMSR2 data',
utils.SUMMARY_FIELDS['processing_level']: '3'
})
result = {utils.SUMMARY_FIELDS['description']: 'GCOM-W AMSR2 data'}
processing_level_match = re.match(r'^.*/L([1-3][A-Z]?)(\.[^/]+)?/.*$', raw_metadata['url'])
if processing_level_match:
result[utils.SUMMARY_FIELDS['processing_level']] = processing_level_match.group(1)
return utils.dict_to_string(result)

time_patterns = (
(
Expand All @@ -41,6 +42,15 @@ def get_summary(self, raw_metadata):
utils.create_datetime,
lambda time: (time, time + relativedelta(months=1))
),
(
re.compile(r'/[A-Z\d]+_' +
utils.YEARMONTHDAY_REGEX +
r'(?P<hour>\d{2})' +
r'(?P<minute>\d{2})' +
r'_.*\.h5$'),
utils.create_datetime,
lambda time: (time, time + relativedelta(minutes=50))
),
)

@utils.raises(KeyError)
Expand All @@ -62,6 +72,3 @@ def get_location_geometry(self, raw_metadata):

def get_provider(self, raw_metadata):
return utils.get_gcmd_provider(['JP/JAXA/EOC'])

def get_dataset_parameters(self, raw_metadata):
return utils.create_parameter_list(('sea_surface_temperature',))
36 changes: 21 additions & 15 deletions tests/normalizers/test_gportal_gcom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
from metanorm.errors import MetadataNormalizationError


class GPortalGCOMAMSR2L3MetadataNormalizerTestCase(unittest.TestCase):
class GPortalGCOMWAMSR2MetadataNormalizerTestCase(unittest.TestCase):
"""Tests for the GPortal GCOM-W ftp normalizer"""
def setUp(self):
self.normalizer = normalizers.geospaas.GPortalGCOMAMSR2L3MetadataNormalizer()
self.normalizer = normalizers.geospaas.GPortalGCOMWAMSR2MetadataNormalizer()

def test_check(self):
"""Test the checking condition"""
Expand All @@ -23,16 +23,20 @@ def test_check(self):
'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST_25/3/2018/07/'
'GW1AM2_20180703_01D_EQOA_L3SGSSTHB3300300.h5'
}))
self.assertTrue(self.normalizer.check({
'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L1R/2/2020/02/'
'GW1AM2_202002011046_045A_L1SGRTBR_2220220.h5'
}))

self.assertFalse(self.normalizer.check({}))
self.assertFalse(self.normalizer.check({'url': 'ftp://foo/bar'}))

def test_entry_title(self):
"""entry_title from GPortalGCOMAMSR2L3MetadataNormalizer """
self.assertEqual(self.normalizer.get_entry_title({}), 'AMSR2-L3 Sea Surface Temperature')
"""entry_title from GPortalGCOMWAMSR2MetadataNormalizer """
self.assertEqual(self.normalizer.get_entry_title({}), 'GCOM-W AMSR2')

def test_entry_id(self):
"""entry_id from GPortalGCOMAMSR2L3MetadataNormalizer """
"""entry_id from GPortalGCOMWAMSR2MetadataNormalizer """
attributes = {
'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST_25/3/2012/07/'
'GW1AM2_201207031905_134D_L2SGSSTLB3300300.h5'}
Expand All @@ -49,10 +53,19 @@ def test_entry_id_error(self):
self.normalizer.get_entry_id({})

def test_summary(self):
"""summary from GPortalGCOMAMSR2L3MetadataNormalizer """
"""summary from GPortalGCOMWAMSR2MetadataNormalizer """
self.assertEqual(
self.normalizer.get_summary({}),
self.normalizer.get_summary({
'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L3.SST_25/3/2017/12/'
'GW1AM2_20171201_01D_EQOA_L3SGSSTLB3300300.h5'
}),
'Description: GCOM-W AMSR2 data;Processing level: 3')
self.assertEqual(
self.normalizer.get_summary({
'url': 'ftp://ftp.gportal.jaxa.jp/standard/GCOM-W/GCOM-W.AMSR2/L1R/2/2020/02/'
'GW1AM2_202002011046_045A_L1SGRTBR_2220220.h5'
}),
'Description: GCOM-W AMSR2 data;Processing level: 1R')

def test_time_coverage_start_day(self):
"""Test getting time_coverage_start from a day file"""
Expand Down Expand Up @@ -118,14 +131,7 @@ def test_gcmd_provider(self):
mock_get_gcmd_method.return_value)

def test_location_geometry(self):
"""geometry from GPortalGCOMAMSR2L3MetadataNormalizer """
"""geometry from GPortalGCOMWAMSR2MetadataNormalizer """
self.assertEqual(
self.normalizer.get_location_geometry({}),
'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))')

def test_dataset_parameters(self):
"""dataset_parameters from GPortalGCOMAMSR2L3MetadataNormalizer """
with mock.patch('metanorm.utils.create_parameter_list') as mock_get_gcmd_method:
self.assertEqual(
self.normalizer.get_dataset_parameters({}),
mock_get_gcmd_method.return_value)

0 comments on commit 3c8ea53

Please sign in to comment.