From c2ec46ec884112307f73c731c2aa2528147d460a Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 18 Jan 2023 02:05:15 +0000 Subject: [PATCH 1/3] Refactor entrypoint endpoint to support pagination. Add tern vocabs entrypoint --- src/linkeddata_api/domain/schema.py | 24 ++-- .../domain/viewer/entrypoints/__init__.py | 1 + .../domain/viewer/entrypoints/nrm.py | 81 +++++++++--- .../domain/viewer/entrypoints/tern.py | 122 ++++++++++++++++++ .../domain/viewer/entrypoints/utils.py | 9 ++ .../views/api_v1/viewer/entrypoint.py | 18 ++- 6 files changed, 223 insertions(+), 32 deletions(-) create mode 100644 src/linkeddata_api/domain/viewer/entrypoints/tern.py create mode 100644 src/linkeddata_api/domain/viewer/entrypoints/utils.py diff --git a/src/linkeddata_api/domain/schema.py b/src/linkeddata_api/domain/schema.py index 19fd668..617782b 100644 --- a/src/linkeddata_api/domain/schema.py +++ b/src/linkeddata_api/domain/schema.py @@ -3,14 +3,6 @@ from pydantic import BaseModel -class Item(BaseModel): - id: str - label: str - description: str = None - created: str = None - modified: str = None - - class RDFListItemMixin(BaseModel): """An item in an RDF List""" @@ -56,3 +48,19 @@ class Resource(BaseModel): types: list[URI] properties: list[PredicateObjects] incoming_properties: list[SubjectPredicates] + + +class EntrypointItem(BaseModel): + id: str + label: str + description: str = None + created: str = None + modified: str = None + + +class EntrypointItems(BaseModel): + items: list[EntrypointItem] + more_pages_exists: bool + items_count: int + limit: int + total_pages: int diff --git a/src/linkeddata_api/domain/viewer/entrypoints/__init__.py b/src/linkeddata_api/domain/viewer/entrypoints/__init__.py index 0bc2814..2743009 100644 --- a/src/linkeddata_api/domain/viewer/entrypoints/__init__.py +++ b/src/linkeddata_api/domain/viewer/entrypoints/__init__.py @@ -1,2 +1,3 @@ from . import exceptions from . import nrm +from . import tern diff --git a/src/linkeddata_api/domain/viewer/entrypoints/nrm.py b/src/linkeddata_api/domain/viewer/entrypoints/nrm.py index b129276..e65b722 100644 --- a/src/linkeddata_api/domain/viewer/entrypoints/nrm.py +++ b/src/linkeddata_api/domain/viewer/entrypoints/nrm.py @@ -1,22 +1,54 @@ -from typing import Optional +from jinja2 import Template from linkeddata_api import data from linkeddata_api.domain import schema +from linkeddata_api.domain.viewer.entrypoints.utils import ( + ceiling_division, + get_optional_value, +) -def get_optional_value(row: dict, key: str) -> Optional[str]: - return row.get(key)["value"] if row.get(key) else None +def get_count(sparql_endpoint: str) -> int: + query = """ + PREFIX skos: + PREFIX dcterms: + PREFIX reg: + SELECT + (COUNT(*) AS ?count) + FROM + FROM + WHERE { + dcterms:hasPart ?uri . + VALUES (?vocabularyType) { + (skos:ConceptScheme) + (skos:Collection) + } + ?uri a ?vocabularyType ; + skos:prefLabel ?_label . + + OPTIONAL { ?uri dcterms:description ?_description } + OPTIONAL { ?uri dcterms:created ?_created } + OPTIONAL { ?uri dcterms:modified ?_modified } + } + """ + + result = data.sparql.post(query, sparql_endpoint).json() + + return int(result["results"]["bindings"][0]["count"]["value"]) def get( sparql_endpoint: str, -) -> schema.Item: + page: int, +) -> schema.EntrypointItems: """Get Raises RequestError and SPARQLResultJSONError """ + limit = 20 - query = """ + query = Template( + """ PREFIX skos: PREFIX dcterms: PREFIX reg: @@ -44,25 +76,34 @@ def get( GROUP by ?uri ORDER by ?label """ + ).render(limit=20, offset=(page - 1) * limit) result = data.sparql.post(query, sparql_endpoint).json() + count = get_count(sparql_endpoint) + more_pages_exist = False + if count > (page * limit): + more_pages_exist = True + + total_pages = ceiling_division(count, limit) + vocabs = [] - try: - for row in result["results"]["bindings"]: - vocabs.append( - schema.Item( - id=str(row["uri"]["value"]), - label=str(row["label"]["value"]), - description=get_optional_value(row, "description"), - created=get_optional_value(row, "created"), - modified=get_optional_value(row, "modified"), - ) + for row in result["results"]["bindings"]: + vocabs.append( + schema.EntrypointItem( + id=str(row["uri"]["value"]), + label=str(row["label"]["value"]), + description=get_optional_value(row, "description"), + created=get_optional_value(row, "created"), + modified=get_optional_value(row, "modified"), ) - except KeyError as err: - raise data.exceptions.SPARQLResultJSONError( - f"Unexpected SPARQL result set.\n{result}\n{err}" - ) from err + ) - return vocabs + return schema.EntrypointItems( + items=vocabs, + more_pages_exists=more_pages_exist, + items_count=count, + limit=limit, + total_pages=total_pages, + ) diff --git a/src/linkeddata_api/domain/viewer/entrypoints/tern.py b/src/linkeddata_api/domain/viewer/entrypoints/tern.py new file mode 100644 index 0000000..bbbf8c2 --- /dev/null +++ b/src/linkeddata_api/domain/viewer/entrypoints/tern.py @@ -0,0 +1,122 @@ +from jinja2 import Template + +from linkeddata_api import data +from linkeddata_api.domain import schema +from linkeddata_api.domain.viewer.entrypoints.utils import ( + ceiling_division, + get_optional_value, +) + + +def get_count(sparql_endpoint: str) -> int: + query = """ + PREFIX skos: + PREFIX dcterms: + PREFIX owl: + SELECT + ( + COUNT( + * + ) + AS ?count + ) + FROM + FROM + WHERE { + VALUES (?vocabularyType) { + (skos:ConceptScheme) + (skos:Collection) + } + ?uri a ?vocabularyType ; + skos:prefLabel ?_label . + + OPTIONAL { ?uri dcterms:description ?_description } + OPTIONAL { ?uri dcterms:created ?_created } + OPTIONAL { ?uri dcterms:modified ?_modified } + + FILTER NOT EXISTS { + ?uri owl:deprecated true . + } + } + """ + + result = data.sparql.post(query, sparql_endpoint).json() + + return int(result["results"]["bindings"][0]["count"]["value"]) + + +def get( + sparql_endpoint: str, + page: int, +) -> schema.EntrypointItems: + """Get + + Raises RequestError and SPARQLResultJSONError + """ + limit = 20 + + query = Template( + """ + PREFIX skos: + PREFIX dcterms: + PREFIX owl: + SELECT + ?uri + (SAMPLE(?_label) as ?label) + (SAMPLE(?_description) as ?description) + (SAMPLE(?_created) as ?created) + (SAMPLE(?_modified) as ?modified) + FROM + FROM + WHERE { + VALUES (?vocabularyType) { + (skos:ConceptScheme) + (skos:Collection) + } + ?uri a ?vocabularyType ; + skos:prefLabel ?_label . + + OPTIONAL { ?uri dcterms:description ?_description } + OPTIONAL { ?uri dcterms:created ?_created } + OPTIONAL { ?uri dcterms:modified ?_modified } + + FILTER NOT EXISTS { + ?uri owl:deprecated true . + } + } + GROUP by ?uri + ORDER by ?label + LIMIT {{ limit }} + OFFSET {{ offset }} + """ + ).render(limit=20, offset=(page - 1) * limit) + + result = data.sparql.post(query, sparql_endpoint).json() + + count = get_count(sparql_endpoint) + more_pages_exist = False + if count > (page * limit): + more_pages_exist = True + + total_pages = ceiling_division(count, limit) + + vocabs = [] + + for row in result["results"]["bindings"]: + vocabs.append( + schema.EntrypointItem( + id=str(row["uri"]["value"]), + label=str(row["label"]["value"]), + description=get_optional_value(row, "description"), + created=get_optional_value(row, "created"), + modified=get_optional_value(row, "modified"), + ) + ) + + return schema.EntrypointItems( + items=vocabs, + more_pages_exists=more_pages_exist, + items_count=count, + limit=limit, + total_pages=total_pages, + ) diff --git a/src/linkeddata_api/domain/viewer/entrypoints/utils.py b/src/linkeddata_api/domain/viewer/entrypoints/utils.py new file mode 100644 index 0000000..05707f5 --- /dev/null +++ b/src/linkeddata_api/domain/viewer/entrypoints/utils.py @@ -0,0 +1,9 @@ +from typing import Optional, Union + + +def get_optional_value(row: dict, key: str) -> Optional[str]: + return row.get(key)["value"] if row.get(key) else None + + +def ceiling_division(a: Union[int, float], b: Union[int, float]) -> int: + return int(-(a // -b)) diff --git a/src/linkeddata_api/views/api_v1/viewer/entrypoint.py b/src/linkeddata_api/views/api_v1/viewer/entrypoint.py index 438ca03..5dd39f8 100644 --- a/src/linkeddata_api/views/api_v1/viewer/entrypoint.py +++ b/src/linkeddata_api/views/api_v1/viewer/entrypoint.py @@ -1,4 +1,4 @@ -from flask import abort +from flask import abort, request from flask_tern import openapi from linkeddata_api.views.api_v1.blueprint import bp @@ -15,20 +15,28 @@ "nrm": { "func": domain.viewer.entrypoints.nrm.get, "sparql_endpoint": "https://graphdb.tern.org.au/repositories/dawe_vocabs_core", - } + }, + "tern": { + "func": domain.viewer.entrypoints.tern.get, + "sparql_endpoint": "https://graphdb.tern.org.au/repositories/tern_vocabs_core", + }, } @bp.get("/viewer/entrypoint/") @openapi.validate(validate_request=False, validate_response=False) def get_entrypoint(viewer_id: str): + page = int(request.args.get("page", 1)) + if page < 1: + page = 1 + try: obj = mapping.get(viewer_id) if obj is None: raise ViewerIDNotFoundError(f"Key '{viewer_id}' not found") sparql_endpoint = obj["sparql_endpoint"] - items = obj["func"](sparql_endpoint) + entrypoint_items = obj["func"](sparql_endpoint, page) except ViewerIDNotFoundError as err: abort(404, str(err)) except (RequestError, SPARQLResultJSONError) as err: @@ -36,4 +44,6 @@ def get_entrypoint(viewer_id: str): except Exception as err: abort(500, err) - return jsonify(items, headers={"cache-control": "max-age=600, s-maxage=3600"}) + return jsonify( + entrypoint_items, headers={"cache-control": "max-age=600, s-maxage=3600"} + ) From b6065fd09589cf12f0a1736412a86c40fff612a3 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 18 Jan 2023 05:47:33 +0000 Subject: [PATCH 2/3] Add order by, limit and offset. Add filter not exists. Formatting --- .../domain/viewer/entrypoints/nrm.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/linkeddata_api/domain/viewer/entrypoints/nrm.py b/src/linkeddata_api/domain/viewer/entrypoints/nrm.py index e65b722..a03bb1e 100644 --- a/src/linkeddata_api/domain/viewer/entrypoints/nrm.py +++ b/src/linkeddata_api/domain/viewer/entrypoints/nrm.py @@ -13,8 +13,7 @@ def get_count(sparql_endpoint: str) -> int: PREFIX skos: PREFIX dcterms: PREFIX reg: - SELECT - (COUNT(*) AS ?count) + SELECT (COUNT(*) AS ?count) FROM FROM WHERE { @@ -29,6 +28,10 @@ def get_count(sparql_endpoint: str) -> int: OPTIONAL { ?uri dcterms:description ?_description } OPTIONAL { ?uri dcterms:created ?_created } OPTIONAL { ?uri dcterms:modified ?_modified } + + FILTER NOT EXISTS { + ?uri owl:deprecated true . + } } """ @@ -52,10 +55,10 @@ def get( PREFIX skos: PREFIX dcterms: PREFIX reg: - SELECT + SELECT ?uri - (SAMPLE(?_label) as ?label) - (SAMPLE(?_description) as ?description) + (SAMPLE(?_label) as ?label) + (SAMPLE(?_description) as ?description) (SAMPLE(?_created) as ?created) (SAMPLE(?_modified) as ?modified) FROM @@ -74,7 +77,9 @@ def get( OPTIONAL { ?uri dcterms:modified ?_modified } } GROUP by ?uri - ORDER by ?label + ORDER by ?label + LIMIT {{ limit }} + OFFSET {{ offset }} """ ).render(limit=20, offset=(page - 1) * limit) From 92f823c82f6d485fbb97264f13a1b6423be040c9 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 18 Jan 2023 05:47:47 +0000 Subject: [PATCH 3/3] SPARQL query formatting --- .../domain/viewer/entrypoints/tern.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/linkeddata_api/domain/viewer/entrypoints/tern.py b/src/linkeddata_api/domain/viewer/entrypoints/tern.py index bbbf8c2..9d614ec 100644 --- a/src/linkeddata_api/domain/viewer/entrypoints/tern.py +++ b/src/linkeddata_api/domain/viewer/entrypoints/tern.py @@ -13,13 +13,7 @@ def get_count(sparql_endpoint: str) -> int: PREFIX skos: PREFIX dcterms: PREFIX owl: - SELECT - ( - COUNT( - * - ) - AS ?count - ) + SELECT (COUNT(*) AS ?count) FROM FROM WHERE { @@ -61,11 +55,11 @@ def get( PREFIX dcterms: PREFIX owl: SELECT - ?uri - (SAMPLE(?_label) as ?label) - (SAMPLE(?_description) as ?description) - (SAMPLE(?_created) as ?created) - (SAMPLE(?_modified) as ?modified) + ?uri + (SAMPLE(?_label) as ?label) + (SAMPLE(?_description) as ?description) + (SAMPLE(?_created) as ?created) + (SAMPLE(?_modified) as ?modified) FROM FROM WHERE {