Merge pull request #82 from lpoaura/fix_81

fix pagination calculation (fix #81)
lpoaura · Aug 20, 2024 · 66bd681 · 66bd681
2 parents 6b82cae + 6eebd77
commit 66bd681
Show file tree

Hide file tree

Showing 9 changed files with 151 additions and 67 deletions.
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -63,7 +63,7 @@ jobs:
           retention-days: 30
 
       - name: Setup Pages
-        uses: actions/configure-pages@v4
+        uses: actions/configure-pages@v5
         if: github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main')
 
       - name: Upload artifact

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,9 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
 
 <!-- ## Unreleased [{version_tag}](https://github.com/opengisch/qgis-plugin-ci/releases/tag/{version_tag}) - YYYY-MM-DD -->
 
+## 1.6.6 - 2024-08-20
+
+### Fixes
+
+- Pagination is now calculated from API response (`limit` value), fix #81
+
+## 1.6.5 - 2024-06-10
+
+### Fixes
+
+- Fix trigger to populate additional data (fix #73).
+
 ## 1.6.4 - 2024-04-24
 
-## Fixes
+### Fixes
 
 - Fix typo error on trigger scripts.
 

diff --git a/data/source_samples/geonature_export_sinp_with_metadata.sql b/data/source_samples/geonature_export_sinp_with_metadata.sql
@@ -22,26 +22,73 @@ WITH af_actors AS (SELECT cafa.id_acquisition_framework,
                             LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
                             JOIN ref_nomenclatures.t_nomenclatures tn
                                  ON cafa.id_nomenclature_actor_role = tn.id_nomenclature),
-     af AS (SELECT taf.id_acquisition_framework,
-                   jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
-                                      taf.acquisition_framework_name,
-                                      'desc', taf.acquisition_framework_desc, 'start_date',
-                                      taf.acquisition_framework_start_date, 'end_date',
-                                      taf.acquisition_framework_end_date,
-                                      'initial_closing_date', taf.initial_closing_date, 'territorial_level',
-                                      ntl.cd_nomenclature, 'financing_type', nft.cd_nomenclature, 'target_description',
-                                      taf.target_description, 'ecologic_or_geologic_target',
-                                      taf.ecologic_or_geologic_target, 'actors',
-                                      json_agg(af_actors.json_data)) AS af_data
-            FROM gn_meta.t_acquisition_frameworks taf
-                     JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
-                     LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
-                               ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
-                     LEFT JOIN ref_nomenclatures.t_nomenclatures nft
-                               ON taf.id_nomenclature_financing_type = nft.id_nomenclature
-            GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
-                     taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
-                     ntl.cd_nomenclature, nft.cd_nomenclature),
+     af_territories AS (SELECT caft.id_acquisition_framework,
+                               array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS territories
+                        FROM gn_meta.cor_acquisition_framework_territory caft
+                                 LEFT JOIN ref_nomenclatures.t_nomenclatures
+                                           ON caft.id_nomenclature_territory =
+                                              t_nomenclatures.id_nomenclature
+                        GROUP BY caft.id_acquisition_framework),
+     af_objectives AS (SELECT cafo.id_acquisition_framework,
+                              array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS objectives
+                       FROM gn_meta.cor_acquisition_framework_objectif cafo
+                                LEFT JOIN ref_nomenclatures.t_nomenclatures
+                                          ON cafo.id_nomenclature_objectif =
+                                             t_nomenclatures.id_nomenclature
+                       GROUP BY cafo.id_acquisition_framework),
+     af_voletsinp AS (SELECT cafv.id_acquisition_framework,
+                             array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS voletsinp
+                      FROM gn_meta.cor_acquisition_framework_voletsinp cafv
+                               LEFT JOIN ref_nomenclatures.t_nomenclatures
+                                         ON cafv.id_nomenclature_voletsinp =
+                                            t_nomenclatures.id_nomenclature
+                      GROUP BY cafv.id_acquisition_framework),
+     af_publication AS (SELECT cafp.id_acquisition_framework,
+                               array_agg(DISTINCT
+                                         jsonb_build_object('uuid', sinp_datatype_publications.unique_publication_id,
+                                                            'reference',
+                                                            sinp_datatype_publications.publication_reference,
+                                                            'url',
+                                                            sinp_datatype_publications.publication_url)) AS publications
+                        FROM gn_meta.cor_acquisition_framework_publication cafp
+                                 LEFT JOIN gn_meta.sinp_datatype_publications
+                                           ON cafp.id_publication = sinp_datatype_publications.id_publication
+                        GROUP BY cafp.id_acquisition_framework)
+        ,
+     af AS
+         (SELECT taf.id_acquisition_framework,
+                 jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
+                                    taf.acquisition_framework_name,
+                                    'desc', taf.acquisition_framework_desc, 'start_date',
+                                    taf.acquisition_framework_start_date, 'end_date',
+                                    taf.acquisition_framework_end_date,
+                                    'initial_closing_date', taf.initial_closing_date, 'territories',
+                                    af_territories.territories, 'territorial_level',
+                                    ntl.cd_nomenclature, 'territory_desc', taf.territory_desc, 'objectives',
+                                    af_objectives.objectives, 'publications', af_publication.publications,
+                                    'financing_type', nft.cd_nomenclature,
+                                    'target_description',
+                                    taf.target_description, 'ecologic_or_geologic_target',
+                                    taf.ecologic_or_geologic_target, 'actors',
+                                    json_agg(af_actors.json_data), 'is_parent', taf.is_parent, 'parent_uuid',
+                                    tafp.unique_acquisition_framework_id) AS af_data
+          FROM gn_meta.t_acquisition_frameworks taf
+                   LEFT JOIN gn_meta.t_acquisition_frameworks tafp
+                             ON tafp.id_acquisition_framework = taf.acquisition_framework_parent_id
+                   JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
+                   LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
+                             ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
+                   LEFT JOIN ref_nomenclatures.t_nomenclatures nft
+                             ON taf.id_nomenclature_financing_type = nft.id_nomenclature
+                   LEFT JOIN af_territories ON af_territories.id_acquisition_framework = taf.id_acquisition_framework
+                   LEFT JOIN af_objectives ON af_objectives.id_acquisition_framework = taf.id_acquisition_framework
+                   LEFT JOIN af_voletsinp ON af_voletsinp.id_acquisition_framework = taf.id_acquisition_framework
+                   LEFT JOIN af_publication ON af_publication.id_acquisition_framework = taf.id_acquisition_framework
+          GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
+                   taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
+                   ntl.cd_nomenclature, nft.cd_nomenclature, af_territories.territories, af_objectives.objectives,
+                   af_voletsinp.voletsinp, af_publication.publications, taf.is_parent,
+                   tafp.unique_acquisition_framework_id),
      ds_actors AS (SELECT cda.id_dataset,
                           json_build_object('type_role',
                                             CASE
@@ -63,19 +110,37 @@ WITH af_actors AS (SELECT cafa.id_acquisition_framework,
                             LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
                             JOIN ref_nomenclatures.t_nomenclatures tn
                                  ON cda.id_nomenclature_actor_role = tn.id_nomenclature),
+     ds_protocols AS (SELECT cdp.id_dataset,
+                             jsonb_build_object(
+                                     'uuid', sdp.unique_protocol_id,
+                                     'name', sdp.protocol_name,
+                                     'desc', sdp.protocol_desc,
+                                     'url', sdp.protocol_url,
+                                     'type', t_nomenclatures.cd_nomenclature) AS protocols
+                      FROM gn_meta.cor_dataset_protocol cdp
+                               JOIN gn_meta.sinp_datatype_protocols sdp
+                                    ON cdp.id_protocol = sdp.id_protocol
+                               LEFT JOIN ref_nomenclatures.t_nomenclatures
+                                         ON sdp.id_nomenclature_protocol_type = t_nomenclatures.id_nomenclature)
+        ,
      ds AS (SELECT tds.id_dataset,
                    tds.id_acquisition_framework,
                    --    tds.additional_data,
                    jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
                                       'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
-                                      'collecting_method', ncm.cd_nomenclature, 'data_origin', ndo.cd_nomenclature,
+                                      'keywords', tds.keywords, 'marine_domain', tds.marine_domain,
+                                      'terrestrial_domain', tds.terrestrial_domain, 'collecting_method',
+                                      ncm.cd_nomenclature, 'protocols',
+                                      ds_protocols.protocols,
+                                      'data_origin', ndo.cd_nomenclature,
                                       'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
                                       'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
                                                                                                      ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
                                       'actors', json_agg(ds_actors.json_data)) AS dataset_data
             FROM gn_meta.t_datasets tds
                      JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
                      LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
+                     LEFT JOIN ds_protocols ON ds_protocols.id_dataset = tds.id_dataset
                      LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
                                ON tds.id_nomenclature_data_type = ndt.id_nomenclature
                      LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
@@ -90,7 +155,8 @@ WITH af_actors AS (SELECT cafa.id_acquisition_framework,
                                ON tds.id_nomenclature_source_status = nss.id_nomenclature
             GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
                      tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
-                     ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature)
+                     ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature,
+                     ds_protocols.protocols)
 SELECT s.id_synthese,
        s.entity_source_pk_value                         AS id_source,
        s.unique_id_sinp                                 AS id_perm_sinp,

diff --git a/gn2pg/api.py b/gn2pg/api.py
@@ -137,7 +137,7 @@ def controler(self) -> Optional[str]:
         """Return the controler name."""
         return self._ctrl
 
-    def _url(self, kind: str = "data", params: Optional[dict] = None) -> Optional[str]:
+    def _url(self, kind: str = "data", params: Optional[dict] = None) -> str:
         """Generate export API URL with QueryStrings if params.
 
         Args:
@@ -180,7 +180,7 @@ def page_list(
 
         api_url = self._url(kind, params)
 
-        response = session.get(url=api_url, params={**params, **{"limit": 1}})
+        response = session.get(url=api_url, params={**params})
         logger.debug(
             _("Defining page_list from %s with status code %s"),
             api_url,
@@ -190,15 +190,18 @@ def page_list(
         if response.status_code == 200:
             resp = response.json()
             total_filtered = resp["total_filtered"] if "total_filtered" in resp else resp["total"]
-            total_pages = math.ceil(total_filtered / params.get("limit"))
+            limit = resp["limit"]
+            total_pages = math.ceil(total_filtered / limit)
             logger.debug(
                 _("API %s contains %s data in %s page(s)"),
                 api_url,
                 total_filtered,
                 total_pages,
             )
 
-            page_list = (self._url(kind, {**params, **{"offset": p}}) for p in range(total_pages))
+            page_list = list(
+                self._url(kind, {**params, **{"offset": p}}) for p in range(total_pages)
+            )
             return page_list
         logger.info(_("No data available from from %s"), self._config.name)
         return None

diff --git a/gn2pg/data/to_gnsynthese.sql b/gn2pg/data/to_gnsynthese.sql
@@ -701,7 +701,7 @@ BEGIN
     INTO the_comment_context;
     SELECT new.item #>> '{comment_occurrence}'
     INTO the_comment_description;
-    SELECT new.item #> '{additional_data}'
+    SELECT new.item #> '{donnees_additionnelles}'
     INTO the_additional_data;
     SELECT NULL
     INTO the_meta_validation_date;

diff --git a/gn2pg/metadata.py b/gn2pg/metadata.py
@@ -8,7 +8,7 @@
 PACKAGE = "gn2pg_client"
 PROJECT = "GeoNature 2 PostgreSQL Client application"
 PROJECT_NO_SPACES = PROJECT.replace(" ", "")
-VERSION = "1.6.4"
+VERSION = "1.6.6"
 DESCRIPTION = "Import tool from GeoNature to a PostgreSQL database through Export module API"
 AUTHORS = [
     "@lpofredc (LPOAuRA)",