diff --git a/.github/workflows/tests_and_build.yml b/.github/workflows/tests_and_build.yml index 20e7c456..16b94846 100644 --- a/.github/workflows/tests_and_build.yml +++ b/.github/workflows/tests_and_build.yml @@ -5,7 +5,7 @@ env: BASE_IMAGE_NAME: "${{ secrets.DOCKER_ORG }}/geospaas:2.5.1-slim" IMAGE_NAME_WORKER: "${{ secrets.DOCKER_ORG }}/geospaas_processing_worker" IMAGE_NAME_CLI: "${{ secrets.DOCKER_ORG }}/geospaas_processing_cli" - IDF_CONVERTER_VERSION: '0.0.203' + IDF_CONVERTER_VERSION: '0.1.324' jobs: tests: name: Run unit tests @@ -33,7 +33,7 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} @@ -48,7 +48,7 @@ jobs: - name: Build testing image id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . file: Dockerfile_worker @@ -123,7 +123,7 @@ jobs: "https://api.github.com/repos/nansencenter/idf-converter/contents/idf_converter-${IDF_CONVERTER_VERSION}.tar.gz" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v2 @@ -135,13 +135,13 @@ jobs: ${{ runner.os }}-buildx-testing- - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} - name: Build docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . file: Dockerfile_worker @@ -182,7 +182,7 @@ jobs: echo "::set-output name=VERSION::${TAG}" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v2 @@ -192,13 +192,13 @@ jobs: restore-keys: ${{ runner.os }}-buildx-cli- - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} - name: Build docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . file: Dockerfile_cli diff --git a/Dockerfile_worker b/Dockerfile_worker index a06032e9..27511333 100644 --- a/Dockerfile_worker +++ b/Dockerfile_worker @@ -1,24 +1,35 @@ ARG BASE_IMAGE=nansencenter/geospaas:latest-slim FROM ${BASE_IMAGE} as base -RUN pip3 install --upgrade --no-cache-dir \ +RUN apt update && \ + apt install -y \ + nco + +RUN pip install --upgrade --no-cache-dir \ 'celery==5.2.*' \ 'django-celery-results==2.2.*' \ 'graypy==2.1.*' \ + 'importlib-metadata==4.*' \ + 'netCDF4>=1.6.0' \ 'paramiko<2.9' \ 'redis==4.1.*' \ 'requests_oauthlib==1.3.*' \ 'scp==0.14.*' \ - 'freezegun==1.1.*' + 'freezegun==1.1.*' \ + 'nco' FROM base as full +COPY idf_converter.tar.gz /tmp/idf_converter.tar.gz +RUN pip install /tmp/idf_converter.tar.gz + WORKDIR /tmp/setup COPY setup.py README.md ./ COPY geospaas_processing ./geospaas_processing -RUN python3 setup.py bdist_wheel && \ - pip3 install -v dist/geospaas_processing-*.whl && \ +RUN python setup.py bdist_wheel && \ + pip install -v dist/geospaas_processing-*.whl && \ cd /tmp && rm -rf /tmp/setup/ + WORKDIR / ENTRYPOINT ["celery"] diff --git a/geospaas_processing/converters/idf/converter.py b/geospaas_processing/converters/idf/converter.py index 64c19022..226b2629 100644 --- a/geospaas_processing/converters/idf/converter.py +++ b/geospaas_processing/converters/idf/converter.py @@ -185,6 +185,9 @@ class SingleResultIDFConverter(IDFConverter): ParameterSelector( matches=lambda d: d.entry_id.startswith('GL_TS_DB_'), parameter_files=('cmems_013_030_drifter_0m', 'cmems_013_030_drifter_15m')), + ParameterSelector( + lambda d: re.match(r'^mercatorbiomer4v2r1_global_mean_[0-9]{8}$', d.entry_id), + parameter_files=('cmems_001_028_daily_mean_0.5m',)), ParameterSelector( matches=lambda d: re.match( '^D[0-9]{3}-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2\.1-v02\.0-fv01\.0$', @@ -203,7 +206,8 @@ class SingleResultIDFConverter(IDFConverter): matches=lambda d: '-NAVO-L2P_GHRSST-SST1m-VIIRS' in d.entry_id, parameter_files=('ghrsst_l2p_viirs_navo_sst',)), ParameterSelector( - matches=lambda d: 'OSPO-L2P_GHRSST-SSTsubskin-VIIRS' in d.entry_id, + matches=lambda d: ('OSPO-L2P_GHRSST-SSTsubskin-VIIRS' in d.entry_id or + '-STAR-L2P_GHRSST-SSTsubskin-VIIRS' in d.entry_id), parameter_files=('ghrsst_l2p_viirs_ospo_sst',)), ParameterSelector( matches=lambda d: '-OSISAF-L3C_GHRSST-SSTsubskin-AVHRR_SST_METOP_B_GLB-' in d.entry_id, @@ -278,4 +282,7 @@ class MultiResultFoldersIDFConverter(IDFConverter): ParameterSelector( matches=lambda d: '_hts-CMCC--TEMP-MFSeas6-MEDATL-' in d.entry_id, parameter_files=('mfs_med-cmcc-temp',)), + ParameterSelector( + matches=lambda d: 'mfwamglocep_' in d.entry_id, + parameter_files=('meteofrance_model_mfwam',)), ) diff --git a/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_mean_surface b/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_mean_surface index 50191f47..c2b1a3de 100644 --- a/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_mean_surface +++ b/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_mean_surface @@ -8,7 +8,7 @@ time_coverage_relative_end = +1800 geoloc_at_pixel_center = false global_overrides = processing_level:L4,product_version:1.0,file_version:1.0 - variable_overrides_thetao = valid_min:0.0,valid_max:36.0 - variable_overrides_zos = valid_min:-2.5,valid_max:2.5 - variable_overrides_uo = valid_min:-2.5,valid_max:2.5 - variable_overrides_vo = valid_min:-2.5,valid_max:2.5 + override(thetao) = valid_min:0.0,valid_max:36.0 + override(zos) = valid_min:-2.5,valid_max:2.5 + override(uo) = valid_min:-2.5,valid_max:2.5 + override(vo) = valid_min:-2.5,valid_max:2.5 diff --git a/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_smoc b/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_smoc index 3aea3dda..c838d94f 100644 --- a/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_smoc +++ b/geospaas_processing/converters/idf/parameters/cmems_001_024_hourly_smoc @@ -1,6 +1,5 @@ -t netcdf/grid/latlon --o path = seascope/data - collection = cmems_001_024_hourly_smoc +-o collection = cmems_001_024_hourly_smoc -i time_variable = time lon_variable = longitude lat_variable = latitude @@ -8,16 +7,12 @@ depth_dimension = depth depth_value = 0.5 depth_tolerance = 0.5 - variables = uo,vo,vsdx,vsdy,utide,vtide,utotal,vtotal + variables = uo,vo,utotal,vtotal time_coverage_relative_start = -1800 time_coverage_relative_end = +1800 geoloc_at_pixel_center = false global_overrides = processing_level:L4,product_version:1.0,file_version:1.0 - variable_overrides_uo = valid_min:-2.5,valid_max:2.5 - variable_overrides_vo = valid_min:-2.5,valid_max:2.5 - variable_overrides_vsdx = valid_min:-1.5,valid_max:1.5 - variable_overrides_vsdy = valid_min:-1.5,valid_max:1.5 - variable_overrides_utide = valid_min:-2.5,valid_max:2.5 - variable_overrides_vtide = valid_min:-2.5,valid_max:2.5 - variable_overrides_utotal = valid_min:-2.5,valid_max:2.5 - variable_overrides_vtotal = valid_min:-2.5,valid_max:2.5 + override(uo) = valid_min:-2.5,valid_max:2.5 + override(vo) = valid_min:-2.5,valid_max:2.5 + override(utotal) = valid_min:-2.5,valid_max:2.5 + override(vtotal) = valid_min:-2.5,valid_max:2.5 diff --git a/geospaas_processing/converters/idf/parameters/cmems_001_028_daily_mean_0.5m b/geospaas_processing/converters/idf/parameters/cmems_001_028_daily_mean_0.5m new file mode 100644 index 00000000..bbc74c0c --- /dev/null +++ b/geospaas_processing/converters/idf/parameters/cmems_001_028_daily_mean_0.5m @@ -0,0 +1,14 @@ +-t netcdf/grid/latlon +-o collection = cmems_001_028_daily_mean_0.5m +-i time_variable = time + lon_variable = longitude + lat_variable = latitude + depth_variable = depth + depth_dimension = depth + depth_value = 0.5 + depth_tolerance = 0.5 + variables = chl + time_coverage_relative_start = -43200 + time_coverage_relative_end = +43200 + geoloc_at_pixel_center = false + global_overrides = processing_level:model,product_version:1.,file_version:1.0 diff --git a/geospaas_processing/converters/idf/parameters/cmems_008_046 b/geospaas_processing/converters/idf/parameters/cmems_008_046 index 40d2b9b6..1aaf53af 100644 --- a/geospaas_processing/converters/idf/parameters/cmems_008_046 +++ b/geospaas_processing/converters/idf/parameters/cmems_008_046 @@ -1,8 +1,7 @@ -t netcdf/grid/latlon --o path = seascope/data - collection = cmems_008_046 +-o collection = cmems_008_046 gcp_spacing = 4 --i variables = adt,ugos,vgos,sla,ugosa,vgosa +-i variables = adt,ugos,vgos time_variable = time time_coverage_relative_start = -43200 time_coverage_relative_end = 43200 diff --git a/geospaas_processing/converters/idf/parameters/cmems_013_048_radar_total b/geospaas_processing/converters/idf/parameters/cmems_013_048_radar_total index 614c2ea6..7376705e 100644 --- a/geospaas_processing/converters/idf/parameters/cmems_013_048_radar_total +++ b/geospaas_processing/converters/idf/parameters/cmems_013_048_radar_total @@ -6,8 +6,8 @@ lat_variable = LATITUDE depth_variable = DEPH variables = EWCT,NSCT - variable_overrides_EWCT = flag_variable:QCflag,flag_max:2 - variable_overrides_NSCT = flag_variable:QCflag,flag_max:2 + override(EWCT) = flag_variable:QCflag,flag_max:2 + override(NSCT) = flag_variable:QCflag,flag_max:2 time_coverage_relative_start = -1800 time_coverage_relative_end = +1800 global_overrides = product_version:1.0,file_version:1.0 diff --git a/geospaas_processing/converters/idf/parameters/cmems_015_003_0m b/geospaas_processing/converters/idf/parameters/cmems_015_003_0m index 8c230252..d97ca492 100644 --- a/geospaas_processing/converters/idf/parameters/cmems_015_003_0m +++ b/geospaas_processing/converters/idf/parameters/cmems_015_003_0m @@ -1,6 +1,5 @@ -t netcdf/grid/latlon --o path = seascope/data - collection = cmems_015_003_0m +-o collection = cmems_015_003_0m gcp_spacing = 4 -i variables = uo,vo time_variable = time diff --git a/geospaas_processing/converters/idf/parameters/cmems_015_003_15m b/geospaas_processing/converters/idf/parameters/cmems_015_003_15m index 1e4a2201..26c87c25 100644 --- a/geospaas_processing/converters/idf/parameters/cmems_015_003_15m +++ b/geospaas_processing/converters/idf/parameters/cmems_015_003_15m @@ -1,6 +1,5 @@ -t netcdf/grid/latlon --o path = seascope/data - collection = cmems_015_003_15m +-o collection = cmems_015_003_15m gcp_spacing = 4 -i variables = uo,vo time_variable = time diff --git a/geospaas_processing/converters/idf/parameters/esa_cci_sst b/geospaas_processing/converters/idf/parameters/esa_cci_sst index 188c0def..38a7ddba 100644 --- a/geospaas_processing/converters/idf/parameters/esa_cci_sst +++ b/geospaas_processing/converters/idf/parameters/esa_cci_sst @@ -1,10 +1,9 @@ -t netcdf/grid/latlon --o path = seascope/data - collection = esa_cci_sst +-o collection = esa_cci_sst -i variables = analysed_sst,sea_ice_fraction time_variable = time lon_variable = lon lat_variable = lat global_overrides = time_coverage_end:21000101T000000Z - variable_overrides_analysed_sst = valid_min:270.15,valid_max:315.15 - variable_overrides_sea_ice_fraction = valid_min:0.0,valid_max:1.0 + override(analysed_sst) = valid_min:270.15,valid_max:315.15 + override(sea_ice_fraction) = valid_min:0.0,valid_max:1.0 diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_day b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_day index 23151e83..7be51ad2 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_day +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_day @@ -1,9 +1,9 @@ -t ghrsst/l2p -o collection = ghrsst_l2p_modis_a_day gcp_spacing = 32 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i geoloc_spacing = 64 variables = sea_surface_temperature,chlorophyll_a,K_490 - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto - variable_overrides_chlorophyll_a = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto - variable_overrides_K_490 = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto + override(chlorophyll_a) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto + override(K_490) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_night b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_night index 5bf39d44..b1662c02 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_night +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_modis_a_night @@ -1,7 +1,7 @@ -t ghrsst/l2p -o collection = ghrsst_l2p_modis_a_night gcp_spacing = 32 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i geoloc_spacing = 64 variables = sea_surface_temperature - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_jpl_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_jpl_sst index 75cf0bba..2b39ecc6 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_jpl_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_jpl_sst @@ -1,7 +1,7 @@ -t ghrsst/l2p -o collection=ghrsst_l2p_viirs_jpl_sst gcp_spacing = 32 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:n + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i geoloc_spacing = 64 - variables = sea_surface_temperature,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + variables = sea_surface_temperature + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_navo_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_navo_sst index 0690687d..dc3033aa 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_navo_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_navo_sst @@ -1,7 +1,7 @@ -t ghrsst/l2p -o collection=ghrsst_l2p_viirs_navo_sst gcp_spacing = 32 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i geoloc_spacing = 64 - variables = sea_surface_temperature,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + variables = sea_surface_temperature + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_ospo_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_ospo_sst index 805f83eb..c7e74bc7 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_ospo_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l2p_viirs_ospo_sst @@ -1,7 +1,7 @@ -t ghrsst/l2p -o collection = ghrsst_l2p_viirs_ospo_sst gcp_spacing = 32 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i geoloc_spacing = 128 - variables = sea_surface_temperature,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + variables = sea_surface_temperature + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_avhrr_metop_b_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_avhrr_metop_b_sst index 365130bc..88f50467 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_avhrr_metop_b_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_avhrr_metop_b_sst @@ -1,8 +1,8 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3c_avhrr_metop_b_sst - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i time_variable = time lon_variable = lon lat_variable = lat variables = sea_surface_temperature,l2p_flags,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes16_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes16_sst index 4cd67cde..b9b59300 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes16_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes16_sst @@ -1,9 +1,9 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3c_goes16_sst - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes downscale = yes -i time_variable = time lon_variable = lon lat_variable = lat variables = sea_surface_temperature,l2p_flags,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes17_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes17_sst index 1693118e..74b4c55f 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes17_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_goes17_sst @@ -1,9 +1,9 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3c_goes17_sst - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes downscale = yes -i time_variable = time lon_variable = lon lat_variable = lat variables = sea_surface_temperature,l2p_flags,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_atlantic_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_atlantic_sst index fc17cd6e..205fdb1b 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_atlantic_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_atlantic_sst @@ -1,8 +1,8 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3c_seviri_atlantic_sst - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i time_variable = time lon_variable = lon lat_variable = lat - variables = sea_surface_temperature,l2p_flags,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto + variables = sea_surface_temperature + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:4,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_indian_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_indian_sst index f28221a3..ade618d3 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_indian_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3c_seviri_indian_sst @@ -1,8 +1,8 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3c_seviri_indian_sst - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i time_variable = time lon_variable = lon lat_variable = lat - variables = sea_surface_temperature,l2p_flags,quality_level - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + variables = sea_surface_temperature + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3u_amsr2_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3u_amsr2_sst index b0b6f68d..223605fd 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3u_amsr2_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3u_amsr2_sst @@ -1,7 +1,7 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3u_amsr2_sst gcp_spacing = 4 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i time_variable=time lon_variable=lon lat_variable=lat @@ -9,4 +9,4 @@ time_coverage_relative_start = 0 time_coverage_relative_end = +86400 global_overrides = processing_level:L3,product_version:1.0,file_version:1.0 - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/ghrsst_l3u_gmi_sst b/geospaas_processing/converters/idf/parameters/ghrsst_l3u_gmi_sst index c3dfdd85..9cabffcc 100644 --- a/geospaas_processing/converters/idf/parameters/ghrsst_l3u_gmi_sst +++ b/geospaas_processing/converters/idf/parameters/ghrsst_l3u_gmi_sst @@ -1,7 +1,7 @@ -t netcdf/grid/latlon -o collection = ghrsst_l3u_gmi_sst gcp_spacing = 4 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sea_surface_temperature;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes -i time_variable=time lon_variable=lon lat_variable=lat @@ -9,4 +9,4 @@ time_coverage_relative_start = 0 time_coverage_relative_end = +86400 global_overrides = processing_level:L3,product_version:1.0,file_version:1.0 - variable_overrides_sea_surface_temperature = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto + override(sea_surface_temperature) = flag_variable:quality_level,flag_min:3,valid_min:auto,valid_max:auto diff --git a/geospaas_processing/converters/idf/parameters/hycom_osu b/geospaas_processing/converters/idf/parameters/hycom_osu index 920764af..b6f368ef 100644 --- a/geospaas_processing/converters/idf/parameters/hycom_osu +++ b/geospaas_processing/converters/idf/parameters/hycom_osu @@ -11,7 +11,7 @@ time_coverage_relative_end = +1800 geoloc_at_pixel_center = false global_overrides = processing_level:L4,product_version:1.0,file_version:1.0 - variable_overrides_water_temp = valid_min:0.0,valid_max:36.0 - variable_overrides_surf_el = valid_min:-2.5,valid_max:2.5 - variable_overrides_water_u = valid_min:-2.5,valid_max:2.5 - variable_overrides_water_v = valid_min:-2.5,valid_max:2.5 + override(water_temp) = valid_min:0.0,valid_max:36.0 + override(surf_el) = valid_min:-2.5,valid_max:2.5 + override(water_u) = valid_min:-2.5,valid_max:2.5 + override(water_v) = valid_min:-2.5,valid_max:2.5 diff --git a/geospaas_processing/converters/idf/parameters/ibi_hourly_mean_surface b/geospaas_processing/converters/idf/parameters/ibi_hourly_mean_surface index 1fb91326..299a6206 100644 --- a/geospaas_processing/converters/idf/parameters/ibi_hourly_mean_surface +++ b/geospaas_processing/converters/idf/parameters/ibi_hourly_mean_surface @@ -8,7 +8,7 @@ time_coverage_relative_end = +1800 geoloc_at_pixel_center = false global_overrides = processing_level:L4,product_version:1.0,file_version:1.0 - variable_overrides_thetao = valid_min:0.0,valid_max:36.0 - variable_overrides_zos = valid_min:-2.5,valid_max:2.5 - variable_overrides_uo = valid_min:-2.5,valid_max:2.5 - variable_overrides_vo = valid_min:-2.5,valid_max:2.5 + override(thetao) = valid_min:0.0,valid_max:36.0 + override(zos) = valid_min:-2.5,valid_max:2.5 + override(uo) = valid_min:-2.5,valid_max:2.5 + override(vo) = valid_min:-2.5,valid_max:2.5 diff --git a/geospaas_processing/converters/idf/parameters/meteofrance_model_mfwam b/geospaas_processing/converters/idf/parameters/meteofrance_model_mfwam new file mode 100644 index 00000000..0d72a235 --- /dev/null +++ b/geospaas_processing/converters/idf/parameters/meteofrance_model_mfwam @@ -0,0 +1,4 @@ +-t model/mfwam +-o collection = meteofrance_model_mfwam +-i time_coverage_relative_start = -5400 + time_coverage_relative_end = +5400 diff --git a/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-cur b/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-cur index 89afde97..376992e5 100644 --- a/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-cur +++ b/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-cur @@ -12,5 +12,5 @@ time_coverage_relative_end = +1800 geoloc_at_pixel_center = false global_overrides = processing_level:L4,product_version:1.0,file_version:1.0 - variable_overrides_uo = valid_min:-2.5,valid_max:2.5 - variable_overrides_vo = valid_min:-2.5,valid_max:2.5 + override(uo) = valid_min:-2.5,valid_max:2.5 + override(vo) = valid_min:-2.5,valid_max:2.5 diff --git a/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-temp b/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-temp index 8460d27c..6c60c6e9 100644 --- a/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-temp +++ b/geospaas_processing/converters/idf/parameters/mfs_med-cmcc-temp @@ -12,4 +12,4 @@ time_coverage_relative_end = +1800 geoloc_at_pixel_center = false global_overrides = processing_level:L4,product_version:1.0,file_version:1.0 - variable_overrides_thetao = valid_min:0,valid_max:36 + override(thetao) = valid_min:0,valid_max:36 diff --git a/geospaas_processing/converters/idf/parameters/sentinel3_olci_chl b/geospaas_processing/converters/idf/parameters/sentinel3_olci_chl index 89aba673..84b53102 100644 --- a/geospaas_processing/converters/idf/parameters/sentinel3_olci_chl +++ b/geospaas_processing/converters/idf/parameters/sentinel3_olci_chl @@ -2,4 +2,5 @@ -o collection = sentinel3_olci_chl downscale = yes gcp_spacing = 128 + remove_variables = Oa04_reflectance,Oa06_reflectance,Oa09_reflectance,Oa17_reflectance,CHL_NN,TSM_NN -i geoloc_spacing = 128 diff --git a/geospaas_processing/converters/idf/parameters/sentinel3_slstr_sst b/geospaas_processing/converters/idf/parameters/sentinel3_slstr_sst index 78672911..20ff02e8 100644 --- a/geospaas_processing/converters/idf/parameters/sentinel3_slstr_sst +++ b/geospaas_processing/converters/idf/parameters/sentinel3_slstr_sst @@ -1,6 +1,7 @@ -t sentinel3/slstr/L1/bt -o collection = sentinel3_slstr_sst gcp_spacing = 128 - add_variable_anomaly_sea_surface_temperature = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sst;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no + add_variables(anomaly_sea_surface_temperature) = using:anomaly_from_clim,climatology_path:auxiliary/sst_cci_clim_extrapolated/extra_D*-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_CDR2.1-v02.0-fv01.0.nc,climatology_variable:analysed_sst,targets:sst;,outputs:anomaly_sea_surface_temperature;,default_min:-10,default_max:10,extrapolate:no,interpolate:yes + remove_variables = S7_BT_in,S8_BT_in,S9_BT_in -i sst_coeffs_path = auxiliary/sst_coefficient/S3A_SL_2_N2_CAX_20160216T000000_20991231T235959_20170330T120000___________________MPC_O_AL_004.SEN3/SL_2_N2_CAX.v4.3.1.nc geoloc_spacing = 128 diff --git a/geospaas_processing/converters/syntool/converter.py b/geospaas_processing/converters/syntool/converter.py index 0eb67359..c7d54fbe 100644 --- a/geospaas_processing/converters/syntool/converter.py +++ b/geospaas_processing/converters/syntool/converter.py @@ -150,6 +150,10 @@ class BasicSyntoolConverter(SyntoolConverter): matches=lambda d: d.entry_id.startswith('ice_drift_nh_polstere-'), converter_type='osisaf_sea_ice_drift', ingest_parameter_files='ingest_osisaf_sea_ice_drift',), + ParameterSelector( + matches=lambda d: d.entry_id.startswith('nrt_global_allsat_phy_l4_'), + converter_type='current_cmems_l4', + ingest_parameter_files='ingest_geotiff_4326_vectorfield'), ) def __init__(self, **kwargs): @@ -215,30 +219,6 @@ def run(self, in_file, out_dir, **kwargs): return results -@SyntoolConversionManager.register() -class CMEMSL4CurrentSyntoolConverter(BasicSyntoolConverter): - """Syntool converter for current from CMEMS L4 products""" - PARAMETER_SELECTORS = ( - ParameterSelector( - matches=lambda d: d.entry_id.startswith('nrt_global_allsat_phy_l4_'), - converter_type='current_cmems_l4', - ingest_parameter_files='ingest_geotiff_4326_vectorfield'),) - - def convert(self, in_file, out_dir, options, **kwargs): - converted_files = super().convert(in_file, out_dir, options) - bounding_box = kwargs.pop('bounding_box', None) - if bounding_box: - for converted_file in converted_files: - converted_file_path = Path(out_dir, converted_file) - _, tmp_file = tempfile.mkstemp() - logger.info("Cropping %s with bounding box %s", converted_file, bounding_box) - crop(converted_file_path, tmp_file, bounding_box) - shutil.move(tmp_file, converted_file_path) - else: - logger.info("No bounding box provided for %s", in_file) - return converted_files - - @SyntoolConversionManager.register() class Sentinel1SyntoolConverter(BasicSyntoolConverter): """Syntool converter for Sentinel 1""" diff --git a/geospaas_processing/ops.py b/geospaas_processing/ops.py index 2491bed5..b401b47d 100644 --- a/geospaas_processing/ops.py +++ b/geospaas_processing/ops.py @@ -4,6 +4,9 @@ import logging from enum import Enum +import netCDF4 +from nco import Nco, NCOException +nco = Nco() try: from osgeo import gdal except ImportError: @@ -20,13 +23,62 @@ class OutputFormat(Enum): vrt = 'VRT' -def crop(in_file, out_file, bbox, output_format=OutputFormat.geotiff): +def gdal_crop(in_file, out_file, bbox, output_format=OutputFormat.geotiff): + """Cropping function for files without subdatasets. Uses GDAL. + """ + options = gdal.TranslateOptions( + projWin=tuple(str(b) for b in bbox[0:4]), + format=output_format.value) + gdal.Translate(str(out_file), str(in_file), options=options) + + +def find_netcdf_lon_lat(in_file): + """Try to find the dimension names for longitude and latitude in a + netCDF file + """ + dataset = netCDF4.Dataset(in_file) + longitude_name = '' + latitude_name = '' + + for dimension in dataset.dimensions: + lowercase_dimension = dimension.lower() + if 'lon' in lowercase_dimension: + longitude_name = dimension + if 'lat' in lowercase_dimension: + latitude_name = dimension + + if longitude_name and latitude_name: + return (longitude_name, latitude_name) + else: + raise RuntimeError( + f"Could not determine longitude and latitude dimensions names for {in_file}") + + +def nco_crop(in_file, out_file, bbox): + """Cropping function for netCDF files. Uses NCO. + """ + longitude_name, latitude_name = find_netcdf_lon_lat(in_file) + try: + nco.ncks( + input=in_file, + output=out_file, + options=[ + f"-d {latitude_name},{bbox[3]:f},{bbox[1]:f}", + f"-d {longitude_name},{bbox[0]:f},{bbox[2]:f}", + ]) + except NCOException as error: + raise RuntimeError('An error happened during cropping. Please check that the bounding ' + 'box is within the datasets spatial coverage') from error + + +def crop(in_file, out_file, bbox): """Subset a dataset file using a bounding box. bbox is a sequence containing the limits of the bounding box in the projection of the dataset in the following order: west, north, east, south """ - options = gdal.TranslateOptions( - projWin=tuple(str(b) for b in bbox[0:4]), - format=output_format.value) - return gdal.Translate(str(out_file), str(in_file), options=options) + in_file = str(in_file) + if in_file.endswith('.nc'): + nco_crop(in_file, out_file, bbox) + else: # untested + gdal_crop(in_file, out_file, bbox) diff --git a/geospaas_processing/tasks/core.py b/geospaas_processing/tasks/core.py index e0231748..ac70fa1b 100644 --- a/geospaas_processing/tasks/core.py +++ b/geospaas_processing/tasks/core.py @@ -9,6 +9,7 @@ import celery import celery.utils +import geospaas_processing.ops as ops import geospaas_processing.utils as utils from geospaas_processing.tasks import lock_dataset_files, FaultTolerantTask, WORKING_DIRECTORY from ..downloaders import DownloadManager, TooManyDownloadsError @@ -69,7 +70,7 @@ def archive(self, args): # pylint: disable=unused-argument for file in dataset_files_paths: local_path = os.path.join(WORKING_DIRECTORY, file) logger.info("Compressing %s", local_path) - compressed_file = utils.tar_gzip(local_path) + compressed_file = utils.tar_gzip(local_path, force=True) if compressed_file != local_path: logger.info("Removing %s", local_path) try: @@ -123,3 +124,25 @@ def publish(self, args): # pylint: disable=unused-argument results.append(f"ftp://{ftp_host}/{ftp_path}/{file}") return (dataset_id, results) + + +@app.task(base=FaultTolerantTask, bind=True, track_started=True) +@lock_dataset_files +def crop(self, args, bounding_box=None): + """Downloads the dataset whose ID is `dataset_id`""" + if bounding_box is None: + return args + dataset_id = args[0] + dataset_file_path = args[1][0] + bounding_box_str = '_'.join(str(i) for i in bounding_box) + logger.debug("Cropping dataset %s file '%s' to %s", + dataset_id, dataset_file_path, bounding_box_str) + dataset_file_name, extension = os.path.splitext(os.path.basename(dataset_file_path)) + cropped_file_path = os.path.join( + os.path.dirname(dataset_file_path), + f"{dataset_file_name}_{bounding_box_str}{extension}") + ops.crop( + os.path.join(WORKING_DIRECTORY, dataset_file_path), + os.path.join(WORKING_DIRECTORY, cropped_file_path), + bounding_box) + return (dataset_id, (cropped_file_path,)) diff --git a/geospaas_processing/tasks/idf.py b/geospaas_processing/tasks/idf.py index f7007ded..17b78af7 100644 --- a/geospaas_processing/tasks/idf.py +++ b/geospaas_processing/tasks/idf.py @@ -22,10 +22,10 @@ def convert_to_idf(self, args): # pylint: disable=unused-argument If the file path is `None`, an attempt is made to find a file based on the dataset ID. """ dataset_id = args[0] - dataset_files_paths = args[1][0] - logger.debug("Converting dataset file '%s' to IDF", dataset_files_paths) + dataset_file_path = args[1][0] + logger.debug("Converting dataset file '%s' to IDF", dataset_file_path) converted_files = IDFConversionManager(WORKING_DIRECTORY).convert( - dataset_id, dataset_files_paths) + dataset_id, dataset_file_path) logger.info("Successfully converted '%s' to IDF. The results directores are '%s'", - dataset_files_paths, converted_files) + dataset_file_path, converted_files) return (dataset_id, converted_files) diff --git a/geospaas_processing/utils.py b/geospaas_processing/utils.py index 6aaef153..c7a9c54a 100644 --- a/geospaas_processing/utils.py +++ b/geospaas_processing/utils.py @@ -373,7 +373,7 @@ def is_gzipfile(file_path): with open(file_path, 'rb') as file_handler: return file_handler.read(2) == b'\x1f\x8b' -def tar_gzip(file_path): +def tar_gzip(file_path, force=False): """Makes the file a tar archive compressed with gzip if the file is not one already""" if os.path.isfile(file_path) and (tarfile.is_tarfile(file_path) or zipfile.is_zipfile(file_path) or @@ -381,9 +381,15 @@ def tar_gzip(file_path): return file_path archive_path = f"{file_path}.tar.gz" - if not os.path.isfile(archive_path): - with tarfile.open(archive_path, 'w:gz') as archive: - archive.add(file_path, arcname=os.path.basename(file_path)) + if os.path.isfile(archive_path): + if force: + os.remove(archive_path) + else: + return archive_path + + with tarfile.open(archive_path, 'w:gz') as archive: + archive.add(file_path, arcname=os.path.basename(file_path)) + return archive_path diff --git a/setup.py b/setup.py index 37c3c014..cd2b0cfd 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ 'django-geo-spaas', 'django', 'freezegun', + 'nco', 'oauthlib', 'paramiko', 'PyYAML',