Merge branch 'master' of github.com:inveniosoftware/invenio-rdm-records

inveniosoftware · May 3, 2023 · 382e40b · 382e40b
2 parents 358efd2 + 36a0abf
commit 382e40b
Show file tree

Hide file tree

Showing 22 changed files with 438 additions and 562 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -10,6 +10,11 @@
 Changes
 =======
 
+Version 4.0.0 (released 2023-04-25)
+
+- record: add file metadata to the indexing
+- fixtures: add user locale preferences
+
 Version 3.1.0 (released 2023-04-21)
 
 - assets: move react deposit components

diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
@@ -10,6 +10,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "3.1.0"
+__version__ = "4.0.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")
diff --git a/invenio_rdm_records/config.py b/invenio_rdm_records/config.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2019 CERN.
+# Copyright (C) 2019-2023 CERN.
 # Copyright (C) 2019 Northwestern University.
 # Copyright (C) 2021-2023 Graz University of Technology.
 # Copyright (C) 2023 TU Wien.
@@ -148,6 +148,12 @@ def always_valid(identifier):
             "field": "is_published",
         },
     },
+    "file_type": {
+        "facet": facets.filetype,
+        "ui": {
+            "field": "files.types",
+        },
+    },
     "language": {
         "facet": facets.language,
         "ui": {
@@ -224,7 +230,7 @@ def always_valid(identifier):
 """
 
 RDM_SEARCH = {
-    "facets": ["access_status", "resource_type"],
+    "facets": ["access_status", "file_type", "resource_type"],
     "sort": [
         "bestmatch",
         "newest",
@@ -253,7 +259,7 @@ def always_valid(identifier):
 """
 
 RDM_SEARCH_DRAFTS = {
-    "facets": ["access_status", "is_published", "resource_type"],
+    "facets": ["access_status", "is_published", "resource_type", "file_type"],
     "sort": ["bestmatch", "updated-desc", "updated-asc", "newest", "oldest", "version"],
 }
 """User records search configuration (i.e. list of uploads)."""

diff --git a/invenio_rdm_records/oai.py b/invenio_rdm_records/oai.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021 Graz University of Technology.
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2023 CERN.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -19,7 +19,7 @@
 from invenio_search.engine import dsl
 from lxml import etree
 
-from .proxies import current_rdm_records
+from .proxies import current_rdm_records, current_rdm_records_service
 from .resources.serializers.datacite import DataCite43XMLSerializer
 from .resources.serializers.dcat import DCATSerializer
 from .resources.serializers.dublincore import DublinCoreXMLSerializer
@@ -29,25 +29,28 @@
 
 def dublincore_etree(pid, record, **serializer_kwargs):
     """Get DublinCore XML etree for OAI-PMH."""
-    json = DublinCoreXMLSerializer(**serializer_kwargs).dump_obj(record["_source"])
-    return simpledc.dump_etree(json)
+    item = current_rdm_records_service.oai_result_item(g.identity, record["_source"])
+    # TODO: DublinCoreXMLSerializer should be able to dump an etree directly
+    # instead. See https://github.com/inveniosoftware/flask-resources/issues/117
+    obj = DublinCoreXMLSerializer(**serializer_kwargs).dump_obj(item.to_dict())
+    return simpledc.dump_etree(obj)
 
 
 def oai_marcxml_etree(pid, record):
-    """OAI MARCXML format for OAI-PMH.
-
-    It assumes that record is a search result.
-    """
-    return etree.fromstring(MARCXMLSerializer().serialize_object(record["_source"]))
+    """OAI MARCXML format for OAI-PMH."""
+    item = current_rdm_records_service.oai_result_item(g.identity, record["_source"])
+    # TODO: MARCXMLSerializer should directly be able to dump an etree instead
+    # of internally creating an etree, then dump to xml, then parse into an
+    # etree. See https://github.com/inveniosoftware/flask-resources/issues/117
+    return etree.fromstring(MARCXMLSerializer().serialize_object(item.to_dict()))
 
 
 def oai_dcat_etree(pid, record):
-    """OAI DCAT-AP format for OAI-PMH.
-
-    It assumes that record is a search result.
-    """
+    """OAI DCAT-AP format for OAI-PMH."""
+    item = current_rdm_records_service.oai_result_item(g.identity, record["_source"])
+    # TODO: Ditto. See https://github.com/inveniosoftware/flask-resources/issues/117
     return etree.fromstring(
-        DCATSerializer().serialize_object(record["_source"]).encode(encoding="utf-8")
+        DCATSerializer().serialize_object(item.to_dict()).encode(encoding="utf-8")
     )
 
 
@@ -56,6 +59,7 @@ def datacite_etree(pid, record):
 
     It assumes that record is a search result.
     """
+    # TODO: Ditto. See https://github.com/inveniosoftware/flask-resources/issues/117
     data_dict = DataCite43XMLSerializer().dump_obj(record["_source"])
     return schema43.dump_etree(data_dict)
 
@@ -65,6 +69,8 @@ def oai_datacite_etree(pid, record):
 
     It assumes that record is a search result.
     """
+    # TODO: See https://github.com/inveniosoftware/flask-resources/issues/117
+    # This should be made into a serializer similar to the ones above.
     resource_dict = DataCite43XMLSerializer().dump_obj(record["_source"])
 
     nsmap = {

diff --git a/invenio_rdm_records/records/api.py b/invenio_rdm_records/records/api.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020 CERN.
+# Copyright (C) 2020-2023 CERN.
 # Copyright (C) 2021-2023 TU Wien.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
@@ -88,6 +88,8 @@ class CommonFieldsMixin:
     versions_model_cls = models.RDMVersionsState
     parent_record_cls = RDMParent
 
+    # Remember to update INDEXER_DEFAULT_INDEX in Invenio-App-RDM if you
+    # update the JSONSchema and mappings to a new version.
     schema = ConstantField("$schema", "local://records/record-v6.0.0.json")
 
     dumper = SearchDumper(
@@ -289,7 +291,7 @@ class RDMRecord(CommonFieldsMixin, Record):
 
     files = FilesField(
         store=False,
-        dump=False,
+        dump=True,
         file_cls=RDMFileRecord,
         # Don't create
         create=False,

diff --git a/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json b/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
@@ -439,37 +439,7 @@
       }
     },
     "files": {
-      "type": "object",
-      "description": "Files associated with the record",
-      "additionalProperties": false,
-      "properties": {
-        "enabled": {
-          "type": "boolean",
-          "description": "Set to false for metadata only records."
-        },
-        "default_preview": {
-          "type": "string",
-          "description": "Key of the default previewed file."
-        },
-        "order": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          }
-        },
-        "entries": {
-          "type": "object",
-          "additionalProperties": {
-            "$ref": "local://definitions-v1.0.0.json#/file"
-          }
-        },
-        "meta": {
-          "type": "object",
-          "additionalProperties": {
-            "type": "object"
-          }
-        }
-      }
+      "$ref": "local://definitions-v2.0.0.json#/files-simple"
     },
     "notes": {
       "type": "array",

diff --git a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json
@@ -816,6 +816,54 @@
           },
           "default_preview": {
             "type": "keyword"
+          },
+          "count": {
+            "type": "integer"
+          },
+          "totalbytes": {
+            "type": "integer"
+          },
+          "mimetypes": {
+            "type": "keyword"
+          },
+          "types": {
+            "type": "keyword"
+          },
+          "entries": {
+            "type": "object",
+            "properties": {
+              "uuid": {
+                "enabled": false
+              },
+              "version_id": {
+                "enabled": false
+              },
+              "metadata": {
+                "type": "object",
+                "dynamic": true
+              },
+              "checksum": {
+                "type": "keyword"
+              },
+              "key": {
+                "type": "keyword"
+              },
+              "mimetype": {
+                "type": "keyword"
+              },
+              "size": {
+                "type": "integer"
+              },
+              "ext": {
+                "type": "keyword"
+              },
+              "object_version_id": {
+                "enabled": false
+              },
+              "file_id": {
+                "enabled": false
+              }
+            }
           }
         }
       },

diff --git a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v6.0.0.json b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v6.0.0.json
@@ -767,6 +767,54 @@
           },
           "default_preview": {
             "type": "keyword"
+          },
+          "count": {
+            "type": "integer"
+          },
+          "totalbytes": {
+            "type": "integer"
+          },
+          "mimetypes": {
+            "type": "keyword"
+          },
+          "types": {
+            "type": "keyword"
+          },
+          "entries": {
+            "type": "object",
+            "properties": {
+              "uuid": {
+                "enabled": false
+              },
+              "version_id": {
+                "enabled": false
+              },
+              "metadata": {
+                "type": "object",
+                "dynamic": true
+              },
+              "checksum": {
+                "type": "keyword"
+              },
+              "key": {
+                "type": "keyword"
+              },
+              "mimetype": {
+                "type": "keyword"
+              },
+              "size": {
+                "type": "integer"
+              },
+              "ext": {
+                "type": "keyword"
+              },
+              "object_version_id": {
+                "enabled": false
+              },
+              "file_id": {
+                "enabled": false
+              }
+            }
           }
         }
       },

diff --git a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json
@@ -816,6 +816,54 @@
           },
           "default_preview": {
             "type": "keyword"
+          },
+          "count": {
+            "type": "integer"
+          },
+          "totalbytes": {
+            "type": "integer"
+          },
+          "mimetypes": {
+            "type": "keyword"
+          },
+          "types": {
+            "type": "keyword"
+          },
+          "entries": {
+            "type": "object",
+            "properties": {
+              "uuid": {
+                "enabled": false
+              },
+              "version_id": {
+                "enabled": false
+              },
+              "metadata": {
+                "type": "object",
+                "dynamic": true
+              },
+              "checksum": {
+                "type": "keyword"
+              },
+              "key": {
+                "type": "keyword"
+              },
+              "mimetype": {
+                "type": "keyword"
+              },
+              "size": {
+                "type": "integer"
+              },
+              "ext": {
+                "type": "keyword"
+              },
+              "object_version_id": {
+                "enabled": false
+              },
+              "file_id": {
+                "enabled": false
+              }
+            }
           }
         }
       },