From 22239d8123f8b09541ead44f1f0af1407c4b9c1a Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 11 Dec 2024 14:54:22 -0700 Subject: [PATCH 1/3] MNT: Remove unused recwarn fixture Not needed when using pytest.warns(). --- tests/test_catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 4acf4b6dd..ff90cf95a 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -121,7 +121,7 @@ def test_simple_point_feature_collection_xml(): @recorder.use_cassette('html_then_xml_catalog') -def test_html_link(recwarn): +def test_html_link(): """Test that we fall-back when given an HTML catalog page.""" url = ('http://thredds-test.unidata.ucar.edu/thredds/catalog/' 'grib/NCEP/RAP/CONUS_13km/catalog.html') From e58cf86b53ca16807f3af302af125669b8f379e5 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 11 Dec 2024 15:06:29 -0700 Subject: [PATCH 2/3] MNT: Fix test for parsing GFS NCSS metadata on TDS5 We were incorrectly trying to get logged warnings using recwarn, rather than caplog. Fortunately, the corrected test still passes. --- tests/test_ncss_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ncss_dataset.py b/tests/test_ncss_dataset.py index 592621441..d6636d381 100644 --- a/tests/test_ncss_dataset.py +++ b/tests/test_ncss_dataset.py @@ -576,10 +576,10 @@ def test_dataset_elements_full_ncss_grid(): @recorder.use_cassette('GFS_TDS5') -def test_dataset_parsing_tds5(recwarn): +def test_dataset_parsing_tds5(caplog): """Test parsing the dataset from TDS 5.""" url = ('http://thredds-test.unidata.ucar.edu/thredds/ncss/grid/casestudies/irma/model/' 'gfs/GFS_Global_0p5deg_20170903_1200.grib2/dataset.xml') element = ET.fromstring(session_manager.urlopen(url).read()) NCSSDataset(element) - assert len(recwarn) == 0 + assert len(caplog.records) == 0 \ No newline at end of file From 63eb3aba9faa6aa8d14864b07add8a5c82a8c7d3 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 11 Dec 2024 15:14:35 -0700 Subject: [PATCH 3/3] MNT: Eliminate warnings when parsing NCSS dataset.xml (Fixes #112) Handle shapes being enclosed in []. Also make sure the parsed shapes are correct, including for the case of a scalar being given [] instead of [0]. --- src/siphon/ncss_dataset.py | 5 +- tests/fixtures/RAP_TDS5 | 1199 ++++++++++++++++++++++++++++++++++++ tests/test_ncss_dataset.py | 14 +- 3 files changed, 1216 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/RAP_TDS5 diff --git a/src/siphon/ncss_dataset.py b/src/siphon/ncss_dataset.py index cbfc30211..3137c7a43 100644 --- a/src/siphon/ncss_dataset.py +++ b/src/siphon/ncss_dataset.py @@ -75,7 +75,10 @@ def handle_typed_values(val, type_name, value_type): """ if value_type in ['byte', 'short', 'int', 'long']: try: - val = [int(v) for v in re.split('[ ,]', val) if v] + if val := val.strip('[]'): + val = [int(v) for v in re.split('[ ,]', val) if v] + else: + return [0] except ValueError: log.warning('Cannot convert "%s" to int. Keeping type as str.', val) elif value_type in ['float', 'double']: diff --git a/tests/fixtures/RAP_TDS5 b/tests/fixtures/RAP_TDS5 new file mode 100644 index 000000000..58332c2e8 --- /dev/null +++ b/tests/fixtures/RAP_TDS5 @@ -0,0 +1,1199 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br, zstd + Connection: + - keep-alive + User-Agent: + - Siphon (0.9.post518+gf913ddb3.d20241211) + method: GET + uri: https://thredds.ucar.edu/thredds/ncss/grid/grib/NCEP/RAP/CONUS_13km/RR_CONUS_13km_20241211_1900.grib2/dataset.xml + response: + body: + string: "\r\n\r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n 12750.0\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n 9000.0\r\n + \ \r\n \r\n \r\n \r\n 1500.0 4500.0 + 7500.0 10500.0 13500.0 16500.0\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n 75000.0\r\n + \ \r\n \r\n \r\n \r\n 3000.0\r\n + \ \r\n \r\n + \ \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n 500.0 1500.0\r\n + \ \r\n \r\n \r\n + \ -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n + \ \r\n -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n -3338.9278367553097\r\n + \ 2769.8671976532596\r\n -595.6652378084782\r\n + \ 3968.9997879025814\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n -3338.9278367553097\r\n + \ 2769.8671976532596\r\n -595.6652378084782\r\n + \ 3968.9997879025814\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n -3338.9278367553097\r\n + \ 2769.8671976532596\r\n -595.6652378084782\r\n + \ 3968.9997879025814\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n + \ \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ -3338.9278367553097\r\n 2769.8671976532596\r\n + \ -595.6652378084782\r\n 3968.9997879025814\r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n -3338.9278367553097\r\n + \ 2769.8671976532596\r\n -595.6652378084782\r\n + \ 3968.9997879025814\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n + \ \r\n \r\n \r\n + \ \r\n \r\n \r\n \r\n + \ -139.9699\r\n -57.2685\r\n 16.2086\r\n + \ 58.4149\r\n \r\n \r\n 2024-12-11T19:00:00Z\r\n + \ 2024-12-12T16:00:00Z\r\n \r\n \r\n + \ netcdf3\r\n netcdf4-classic\r\n + \ netcdf4ext\r\n \r\n\r\n" + headers: + Connection: + - Keep-Alive + Content-Encoding: + - gzip + Content-Language: + - en-US + Content-Security-Policy: + - 'default-src ''none''; base-uri ''self'' https://www.opendap.org; form-action + ''self''; frame-ancestors ''self''; frame-src ''self''; object-src ''none''; + img-src ''self'' data: https://a.tile.openstreetmap.org https://b.tile.openstreetmap.org + https://c.tile.openstreetmap.org https://www.unidata.ucar.edu https://ahocevar.com; + style-src ''self'' ''unsafe-inline'' https://necolas.github.io https://cdnjs.cloudflare.com; + font-src ''self''; connect-src ''self''; media-src ''self''; manifest-src + ''self''; worker-src ''self''; script-src ''self'' ''unsafe-inline'' ''unsafe-eval'' + https://cdnjs.cloudflare.com https://cdn.rawgit.com https://cdn.jsdelivr.net' + Content-Type: + - application/xml;charset=UTF-8 + Date: + - Wed, 11 Dec 2024 21:51:16 GMT + Keep-Alive: + - timeout=5, max=100 + Server: + - Apache + Strict-Transport-Security: + - max-age=63072000; includeSubdomains; + Transfer-Encoding: + - chunked + Vary: + - Accept-Encoding + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - 1; mode=block + status: + code: 200 + message: '200' +version: 1 diff --git a/tests/test_ncss_dataset.py b/tests/test_ncss_dataset.py index d6636d381..12f64c874 100644 --- a/tests/test_ncss_dataset.py +++ b/tests/test_ncss_dataset.py @@ -582,4 +582,16 @@ def test_dataset_parsing_tds5(caplog): 'gfs/GFS_Global_0p5deg_20170903_1200.grib2/dataset.xml') element = ET.fromstring(session_manager.urlopen(url).read()) NCSSDataset(element) - assert len(caplog.records) == 0 \ No newline at end of file + assert len(caplog.records) == 0 + + +@recorder.use_cassette('RAP_TDS5') +def test_dataset_parsing_rap_tds5(caplog): + """Test parsing the dataset for RAP from TDS5.""" + url = ('https://thredds.ucar.edu/thredds/ncss/grid/grib/NCEP/RAP/CONUS_13km/' + 'RR_CONUS_13km_20241211_1900.grib2/dataset.xml') + element = ET.fromstring(session_manager.urlopen(url).read()) + ds = NCSSDataset(element) + assert len(caplog.records) == 0 + assert ds.axes['y']['shape'] == [337] + assert ds.axes['reftime']['shape'] == [0]