From dbfcb76f0e9a9c6bef3cdce49c934d6e01319c13 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Mon, 23 Jan 2023 02:41:57 +0000 Subject: [PATCH 1/9] Add initial v2 api with pagination and infinite loading working. --- src/linkeddata_api/app.py | 3 +- .../domain/viewer/resource/__init__.py | 8 +- .../domain/viewer/resource/json/__init__.py | 16 +- .../resource/json/profiles/custom_profiles.py | 4 +- src/linkeddata_api/views/api_v2/__init__.py | 7 + src/linkeddata_api/views/api_v2/blueprint.py | 3 + .../views/api_v2/ontology_viewer/__init__.py | 1 + .../ontology_viewer/classes/__init__.py | 1 + .../ontology_viewer/classes/flat/__init__.py | 18 + .../ontology_viewer/classes/flat/crud.py | 89 ++++ .../ontology_viewer/classes/flat/schema.py | 6 + src/linkeddata_api/views/api_v2/openapi.yaml | 439 ++++++++++++++++++ .../views/api_v2/rdf_tools/__init__.py | 1 + .../api_v2/rdf_tools/convert/__init__.py | 25 + .../views/api_v2/version_info.py | 10 + .../views/api_v2/viewer/__init__.py | 3 + .../views/api_v2/viewer/json_renderer.py | 211 +++++++++ .../views/api_v2/viewer/route_entrypoint.py | 49 ++ .../api_v2/viewer/route_predicate_values.py | 47 ++ .../views/api_v2/viewer/route_resource.py | 57 +++ .../views/api_v2/viewer/schema.py | 46 ++ src/linkeddata_api/views/home/__init__.py | 2 +- 22 files changed, 1030 insertions(+), 16 deletions(-) create mode 100644 src/linkeddata_api/views/api_v2/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/blueprint.py create mode 100644 src/linkeddata_api/views/api_v2/ontology_viewer/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/ontology_viewer/classes/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/crud.py create mode 100644 src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/schema.py create mode 100644 src/linkeddata_api/views/api_v2/openapi.yaml create mode 100644 src/linkeddata_api/views/api_v2/rdf_tools/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/rdf_tools/convert/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/version_info.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/json_renderer.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/route_entrypoint.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/route_resource.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/schema.py diff --git a/src/linkeddata_api/app.py b/src/linkeddata_api/app.py index cd203c9..308d886 100644 --- a/src/linkeddata_api/app.py +++ b/src/linkeddata_api/app.py @@ -116,10 +116,11 @@ def create_app(config=None) -> Flask: app.register_blueprint(oidc_login, url_prefix="/api/oidc") # register api blueprints - from linkeddata_api.views import api_v1, home + from linkeddata_api.views import api_v1, api_v2, home app.register_blueprint(home.bp, url_prefix="/api") app.register_blueprint(api_v1.bp, url_prefix="/api/v1.0") + app.register_blueprint(api_v2.bp, url_prefix="/api/v2.0") # setup build_only route so that we can use url_for("root", _external=True) - "root" route required by oidc session login # app.add_url_rule("/", "root", build_only=True) diff --git a/src/linkeddata_api/domain/viewer/resource/__init__.py b/src/linkeddata_api/domain/viewer/resource/__init__.py index 42dfece..4460831 100644 --- a/src/linkeddata_api/domain/viewer/resource/__init__.py +++ b/src/linkeddata_api/domain/viewer/resource/__init__.py @@ -9,7 +9,7 @@ from . import json -def _handle_json_response(uri: str, sparql_endpoint: str) -> str: +def handle_json_response(uri: str, sparql_endpoint: str) -> str: try: result = json.get(uri, sparql_endpoint) except (RequestError, SPARQLNotFoundError, SPARQLResultJSONError) as err: @@ -18,7 +18,7 @@ def _handle_json_response(uri: str, sparql_endpoint: str) -> str: return result.json() -def _handle_rdf_response( +def handle_rdf_response( uri: str, sparql_endpoint: str, format_: str, include_incoming_relationships: bool ) -> str: try: @@ -56,9 +56,9 @@ def get( """ if format_ == "application/json": - result = _handle_json_response(uri, sparql_endpoint) + result = handle_json_response(uri, sparql_endpoint) else: - result = _handle_rdf_response( + result = handle_rdf_response( uri, sparql_endpoint, format_, include_incoming_relationships ) return result diff --git a/src/linkeddata_api/domain/viewer/resource/json/__init__.py b/src/linkeddata_api/domain/viewer/resource/json/__init__.py index d9d9fe0..b64f40e 100644 --- a/src/linkeddata_api/domain/viewer/resource/json/__init__.py +++ b/src/linkeddata_api/domain/viewer/resource/json/__init__.py @@ -103,7 +103,7 @@ def _add_rows_for_rdf_list_items(result: dict, uri: str, sparql_endpoint: str) - @log_time -def _get_uri_label_index( +def get_uri_label_index( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> dict[str, str]: uri_values, _ = _get_uri_values_and_list_items(result, sparql_endpoint, uri) @@ -112,7 +112,7 @@ def _get_uri_label_index( @log_time -def _get_uri_internal_index( +def get_uri_internal_index( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> dict[str, str]: uri_values, _ = _get_uri_values_and_list_items(result, sparql_endpoint, uri) @@ -139,7 +139,7 @@ def get(uri: str, sparql_endpoint: str) -> domain.schema.Resource: result = _add_rows_for_rdf_list_items(result, uri, sparql_endpoint) label = domain.label.get(uri, sparql_endpoint) or uri - types, properties = _get_types_and_properties(result, sparql_endpoint, uri) + types, properties = get_types_and_properties(result, sparql_endpoint, uri) profile_uri = "" ProfileClass = None @@ -187,8 +187,8 @@ def _get_incoming_properties(uri: str, sparql_endpoint: str): sparql_endpoint, ).json() - uri_label_index = _get_uri_label_index(result, sparql_endpoint, uri) - uri_internal_index = _get_uri_internal_index(result, sparql_endpoint, uri) + uri_label_index = get_uri_label_index(result, sparql_endpoint, uri) + uri_internal_index = get_uri_internal_index(result, sparql_endpoint, uri) incoming_properties = [] @@ -232,7 +232,7 @@ def _get_incoming_properties(uri: str, sparql_endpoint: str): @log_time -def _get_types_and_properties( +def get_types_and_properties( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> tuple[list[domain.schema.URI], list[domain.schema.PredicateObjects]]: @@ -242,10 +242,10 @@ def _get_types_and_properties( ] = defaultdict(set) # An index of URIs with label values. - uri_label_index = _get_uri_label_index(result, sparql_endpoint, uri) + uri_label_index = get_uri_label_index(result, sparql_endpoint, uri) # An index of all the URIs linked to and from this resource that are available internally. - uri_internal_index = _get_uri_internal_index(result, sparql_endpoint, uri) + uri_internal_index = get_uri_internal_index(result, sparql_endpoint, uri) if not uri_internal_index.get(uri) and uri is not None: raise data.exceptions.SPARQLNotFoundError(f"Resource with URI {uri} not found.") diff --git a/src/linkeddata_api/domain/viewer/resource/json/profiles/custom_profiles.py b/src/linkeddata_api/domain/viewer/resource/json/profiles/custom_profiles.py index 6f3b5f4..fc4bf24 100644 --- a/src/linkeddata_api/domain/viewer/resource/json/profiles/custom_profiles.py +++ b/src/linkeddata_api/domain/viewer/resource/json/profiles/custom_profiles.py @@ -49,9 +49,9 @@ def _uri(self) -> str: @staticmethod def _process_sparql_values(results: dict) -> list[PredicateObjects]: - from linkeddata_api.domain.viewer.resource.json import _get_types_and_properties + from linkeddata_api.domain.viewer.resource.json import get_types_and_properties - _, properties = _get_types_and_properties( + _, properties = get_types_and_properties( results, "https://graphdb.tern.org.au/repositories/dawe_vocabs_core", ) diff --git a/src/linkeddata_api/views/api_v2/__init__.py b/src/linkeddata_api/views/api_v2/__init__.py new file mode 100644 index 0000000..e9f597e --- /dev/null +++ b/src/linkeddata_api/views/api_v2/__init__.py @@ -0,0 +1,7 @@ +from .blueprint import bp + +# import all sub modules with views registered with blueprint +from . import ontology_viewer +from . import version_info +from . import rdf_tools +from . import viewer diff --git a/src/linkeddata_api/views/api_v2/blueprint.py b/src/linkeddata_api/views/api_v2/blueprint.py new file mode 100644 index 0000000..1729272 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/blueprint.py @@ -0,0 +1,3 @@ +from flask_tern.openapi import OpenApiBlueprint + +bp = OpenApiBlueprint("api_v2", __name__) diff --git a/src/linkeddata_api/views/api_v2/ontology_viewer/__init__.py b/src/linkeddata_api/views/api_v2/ontology_viewer/__init__.py new file mode 100644 index 0000000..eb81565 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/ontology_viewer/__init__.py @@ -0,0 +1 @@ +from . import classes diff --git a/src/linkeddata_api/views/api_v2/ontology_viewer/classes/__init__.py b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/__init__.py new file mode 100644 index 0000000..3471fcb --- /dev/null +++ b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/__init__.py @@ -0,0 +1 @@ +from . import flat diff --git a/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/__init__.py b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/__init__.py new file mode 100644 index 0000000..34101e6 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/__init__.py @@ -0,0 +1,18 @@ +from flask import request +from flask_tern import openapi +from flask_tern.logging import create_audit_event, log_audit + +from linkeddata_api.domain.pydantic_jsonify import jsonify +from linkeddata_api.views.api_v2.blueprint import bp +from . import crud + + +@bp.route("/ontology_viewer/classes/flat") +@openapi.validate(validate_response=False) +def classes_flat_get(): + # TODO: add log audit. + + ontology_id = request.args.get("ontology_id") + classes = crud.get(ontology_id) + + return jsonify(classes) diff --git a/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/crud.py b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/crud.py new file mode 100644 index 0000000..ef602b4 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/crud.py @@ -0,0 +1,89 @@ +from typing import List + +import requests +from jinja2 import Template +from werkzeug.exceptions import HTTPException +from werkzeug.wrappers import Response +from flask_tern.cache import cache + +from . import schema + + +ontology_id_mapping = { + "tern-ontology": { + "sparql_endpoint": "https://graphdb.tern.org.au/repositories/knowledge_graph_core", + "named_graph": "https://w3id.org/tern/ontologies/tern/", + } +} + + +query_template = Template( + """ +# Get a list of classes ordered by label. +# Only one string or langString of type "en" is retrieved as label. + +PREFIX rdfs: +PREFIX owl: +PREFIX sh: +PREFIX skos: + +SELECT distinct ?id (SAMPLE(?_label) as ?label) +FROM +FROM <{{ named_graph }}> +WHERE { + { + ?_class a sh:NodeShape . + ?_class sh:targetClass ?id . + + FILTER(!isBlank(?id)) + + { + ?id rdfs:label ?_label . + } + UNION { + ?id skos:prefLabel ?_label . + } + } +} +GROUP BY ?id +ORDER BY ?label +""" +) + + +@cache.memoize() +def get(ontology_id: str) -> List[schema.ClassItem]: + try: + mapping = ontology_id_mapping[ontology_id] + except KeyError as err: + description = f"Unknown ontology ID '{ontology_id}'. Valid ontology IDs: {list(ontology_id_mapping.keys())}" + raise HTTPException( + description=description, response=Response(description, status=404) + ) from err + + query = query_template.render(named_graph=mapping["named_graph"]) + headers = { + "accept": "application/sparql-results+json", + "content-type": "application/sparql-query", + } + + r = requests.post( + url=mapping["sparql_endpoint"], headers=headers, data=query, timeout=60 + ) + + try: + r.raise_for_status() + except requests.exceptions.HTTPError as err: + raise HTTPException( + description=err.response.text, + response=Response(err.response.text, status=502), + ) from err + + resultset = r.json() + classes = [] + for row in resultset["results"]["bindings"]: + classes.append( + schema.ClassItem(id=row["id"]["value"], label=row["label"]["value"]) + ) + + return classes diff --git a/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/schema.py b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/schema.py new file mode 100644 index 0000000..e57023b --- /dev/null +++ b/src/linkeddata_api/views/api_v2/ontology_viewer/classes/flat/schema.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class ClassItem(BaseModel): + id: str + label: str diff --git a/src/linkeddata_api/views/api_v2/openapi.yaml b/src/linkeddata_api/views/api_v2/openapi.yaml new file mode 100644 index 0000000..494a2ec --- /dev/null +++ b/src/linkeddata_api/views/api_v2/openapi.yaml @@ -0,0 +1,439 @@ +openapi: "3.0.3" + +info: + title: TERN's Linked Data Services API + description: A set of web APIs to power TERN's Linked Data Services website. + version: "2.0" + license: + name: Creative Commons 4.0 + url: https://creativecommons.org/version4 + termsOfService: https://www.tern.org.au/datalicence/ + contact: + email: esupport@tern.org.au + name: TERN eSupport + url: https://ternaus.atlassian.net/wiki/spaces/TERNSup/overview +# servers: +# # base path for api +# # e.g. swagger ui will be at /api/v1.0/ui/ +# - url: /api/v1.0 + +components: + securitySchemes: + BasicAuth: + type: http + scheme: basic + BearerAuth: + type: http + scheme: bearer + OpenID: + type: openIdConnect + # TODO: template this url + openIdConnectUrl: https://auth-test.tern.org.au/auth/realms/local/.well-known/openid-configuration + # TODO: could also define api key via custom header, cookie or url parameter + ApiKeyAuth: + type: apiKey + # TODO: openapi-core validates against hard coded scheme + # scheme: apikey-v1 + # TODO: could also just use BasicAuth auth scheme for apiKey?? -> need to parse basic auth header accordingly to see whether it's user:pw or apikey + in: header # can be "header", "query" or "cookie" + name: Authorization + schemas: + EntrypointItemList: + title: EntrypointItemList + type: array + items: + title: Entrypoint items + type: object + properties: + id: + title: IRI of item + type: string + label: + title: Label of item + type: string + description: + title: The description of the item + type: string + created: + title: The date when the resource was created + oneOf: + - type: string + nullable: true + modified: + title: The date when the resource was modified + oneOf: + - type: string + nullable: true + Resource: + title: Resource + type: object + properties: + uri: + type: string + label: + type: string + types: + type: array + items: + $ref: "#/components/schemas/URI" + profile: + type: string + properties: + type: array + items: + $ref: "#/components/schemas/PredicateObjects" + ClassItem: + title: ClassItem + type: object + properties: + id: + title: IRI of class + type: string + label: + title: Label of class + type: string + URI: + title: URI + type: object + properties: + type: + type: string + label: + type: string + value: + type: string + internal: + type: boolean + list_item: + type: boolean + list_item_number: + oneOf: + - type: number + nullable: true + Literal: + title: Literal + type: object + properties: + type: + type: string + value: + type: string + datatype: + type: string + language: + type: string + PredicateObjects: + title: Predicate objects + type: object + properties: + predicate: + type: object + $ref: "#/components/schemas/URI" + objects: + type: array + items: + oneOf: + - $ref: "#/components/schemas/URI" + - $ref: "#/components/schemas/Literal" + +security: + - BasicAuth: [] + - BearerAuth: [] + # colud define list of scopes here + - OpenID: [] + - ApiKeyAuth: [] + +paths: + /version: + get: + tags: + - General + summary: Application version + description: Get the application's version + responses: + "200": + description: The application's version. + content: + plain/text: + schema: + type: string + /viewer/entrypoint/{viewer_id}: + get: + tags: + - Linked Data viewer + summary: Get the viewer's entrypoint data + parameters: + - in: path + name: viewer_id + schema: + type: string + required: true + examples: + nrm: + value: nrm + responses: + "200": + description: A list of entrypoint items + content: + application/json: + schema: + $ref: "#/components/schemas/EntrypointItemList" + "404": + description: The supplied `viewer_id` value was not found + "502": + description: Gateway error + /viewer/resource: + get: + tags: + - Linked Data viewer + summary: Get RDF resource + description: Get an RDF resource by its URI in an RDF4J repository. + parameters: + - in: query + name: sparql_endpoint + schema: + type: string + required: true + description: The SPARQL endpoint for querying. + examples: + nrm_vocabs: + summary: NRM vocabs SPARQL endpoint + value: https://graphdb.tern.org.au/repositories/dawe_vocabs_core + - in: query + name: uri + schema: + type: string + required: true + description: The URI of the resource. + examples: + nrm_index: + summary: NRM vocab URI + value: https://linked.data.gov.au/def/nrm + nrm_feature_types: + summary: NRM feature types collection + value: https://linked.data.gov.au/def/nrm/31a9f83d-9c8b-4d68-8dd7-d1b7a9a4197b + - in: query + name: format + schema: + type: string + description: The format of the response value. This takes precedence over the request accept header. + examples: + text/turtle: + value: text/turtle + application/n-triples: + value: application/n-triples + application/json: + value: application/json + application/ld+json: + value: application/ld+json + - in: query + name: include_incoming_relationships + schema: + type: string + description: Include incoming relationships. This defaults to `false` if the `format` query parameter is `application/json`. + examples: + true: + value: true + false: + value: false + responses: + "200": + description: RDF resource + content: + application/n-triples: + schema: + type: string + text/turtle: + schema: + type: string + application/ld+json: + schema: + type: string + application/json: + schema: + $ref: "#/components/schemas/Resource" + "400": + description: Client error + "404": + description: Resource of URI not found. + "500": + description: Internal server error + "502": + description: Error communicating with the database. + /viewer/predicate-values: + get: + tags: + - Linked Data viewer + summary: Get RDF resource's predicate values + description: Get an RDF resource's values for one of its predicates. + parameters: + - in: query + name: sparql_endpoint + schema: + type: string + required: true + description: The SPARQL endpoint for querying. + examples: + nrm_vocabs: + summary: NRM vocabs SPARQL endpoint + value: https://graphdb.tern.org.au/repositories/dawe_vocabs_core + - in: query + name: uri + schema: + type: string + required: true + description: The URI of the resource. + examples: + nrm_index: + summary: NRM vocab URI + value: https://linked.data.gov.au/def/nrm + nrm_feature_types: + summary: NRM feature types collection + value: https://linked.data.gov.au/def/nrm/31a9f83d-9c8b-4d68-8dd7-d1b7a9a4197b + - in: query + name: predicate + schema: + type: string + required: true + description: The URI of the resource's predicate + examples: + skos_member: + summary: skos:member + value: http://www.w3.org/2004/02/skos/core#member + - in: query + name: page + schema: + type: integer + description: The page number for the predicate's values. Defaults to 1. + responses: + "200": + description: RDF resource + content: + application/n-triples: + schema: + type: string + text/turtle: + schema: + type: string + application/ld+json: + schema: + type: string + application/json: + schema: + $ref: "#/components/schemas/Resource" + "400": + description: Client error + "404": + description: Resource of URI not found. + "500": + description: Internal server error + "502": + description: Error communicating with the database. + + /ontology_viewer/classes/flat: + get: + tags: + - Ontology viewer + summary: Get a flat list of classes + description: Get a flat list of classes ordered by label. + parameters: + - in: query + name: ontology_id + schema: + type: string + required: true + description: The ontology ID internally known to the Linked Data API. + examples: + tern_ontology: + summary: Request classes from the TERN Ontology + value: tern-ontology + unknown_id: + summary: An unknown ontology ID + value: non-existent-id + responses: + "200": + description: A list of classes ordered by label. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ClassItem" + "404": + description: Unknown ontology ID + content: + text/plain: + schema: + type: string + "502": + description: Gateway error + + /rdf_tools/convert: + post: + tags: + - RDF Tools + summary: Convert from one RDF format to another + description: This uses Python's RDFLib to convert. + # parameters: + # - in: header + # name: accept + # required: true + # schema: + # type: string + # enum: + # - text/turtle + # - application/ld+json + requestBody: + description: The payload data + required: true + content: + application/ld+json: + schema: + type: string + example: + "@context": + "name": "http://schema.org/name" + "image": + "@id": "http://schema.org/image" + "@type": "@id" + "homepage": + "@id": "http://schema.org/url" + "@type": "@id" + "name": "Manu Sporny" + "homepage": "http://manu.sporny.org/" + "image": "http://manu.sporny.org/images/manu.png" + text/turtle: + schema: + type: string + example: | + @prefix ns1: . + + [] ns1:image ; + ns1:name "Manu Sporny" ; + ns1:url . + responses: + "200": + description: Convert + content: + text/turtle: + example: | + @prefix ns1: . + + [] ns1:image ; + ns1:name "Manu Sporny" ; + ns1:url . + application/ld+json: + schema: + type: string + example: + "@context": + "name": "http://schema.org/name" + "image": + "@id": "http://schema.org/image" + "@type": "@id" + "homepage": + "@id": "http://schema.org/url" + "@type": "@id" + "name": "Manu Sporny" + "homepage": "http://manu.sporny.org/" + "image": "http://manu.sporny.org/images/manu.png" \ No newline at end of file diff --git a/src/linkeddata_api/views/api_v2/rdf_tools/__init__.py b/src/linkeddata_api/views/api_v2/rdf_tools/__init__.py new file mode 100644 index 0000000..99a9527 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/rdf_tools/__init__.py @@ -0,0 +1 @@ +from . import convert diff --git a/src/linkeddata_api/views/api_v2/rdf_tools/convert/__init__.py b/src/linkeddata_api/views/api_v2/rdf_tools/convert/__init__.py new file mode 100644 index 0000000..a3a6a3d --- /dev/null +++ b/src/linkeddata_api/views/api_v2/rdf_tools/convert/__init__.py @@ -0,0 +1,25 @@ +import json + +from flask import request, Response +from flask_tern import openapi +from flask_tern.logging import create_audit_event, log_audit + +from linkeddata_api.domain.pydantic_jsonify import jsonify +from linkeddata_api.views.api_v2.blueprint import bp +from linkeddata_api import rdf + + +@bp.route("/rdf_tools/convert", methods=["POST"]) +@openapi.validate(validate_request=False, validate_response=False) +def rdf_tools_convert(): + # TODO: add log audit. + + content_type = request.headers.get("content-type") + accept = request.headers.get("accept") + data = request.data.decode("utf-8") + + graph = rdf.create_graph() + graph.parse(data=data, format=content_type) + + response_data = graph.serialize(format=accept) + return Response(response_data, mimetype=accept) diff --git a/src/linkeddata_api/views/api_v2/version_info.py b/src/linkeddata_api/views/api_v2/version_info.py new file mode 100644 index 0000000..03516c0 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/version_info.py @@ -0,0 +1,10 @@ +from flask import current_app +from flask_tern import openapi + +from .blueprint import bp + + +@bp.route("/version") +@openapi.validate(validate_response=False) +def version_get(): + return current_app.config["VERSION"] diff --git a/src/linkeddata_api/views/api_v2/viewer/__init__.py b/src/linkeddata_api/views/api_v2/viewer/__init__.py new file mode 100644 index 0000000..170e53e --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/__init__.py @@ -0,0 +1,3 @@ +from . import route_resource +from . import route_entrypoint +from . import route_predicate_values diff --git a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py new file mode 100644 index 0000000..6634460 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py @@ -0,0 +1,211 @@ +from jinja2 import Template +from rdflib import RDF + +from linkeddata_api import domain +from linkeddata_api.data import sparql +from linkeddata_api.data.exceptions import SPARQLNotFoundError + +from .schema import URI, Resource, PredicateValues + + +def get_predicate_count_index(uri: str, predicate: str, sparql_endpoint: str) -> int: + query = Template( + """ + SELECT (COUNT(DISTINCT(?value)) as ?count) + WHERE { + <{{ uri }}> <{{ predicate }}> ?value . + } + """ + ).render(uri=uri, predicate=predicate) + + response = sparql.post(query, sparql_endpoint) + + count = int(response.json()["results"]["bindings"][0]["count"]["value"]) + return count + + +def get_predicate_values( + uri: str, predicate: str, sparql_endpoint: str, limit: int, page: int +) -> PredicateValues: + query = Template( + """ + PREFIX rdf: + PREFIX skos: + SELECT ?p ?o ?listItem ?listItemNumber (SAMPLE(?_label) AS ?label) + WHERE { + BIND(<{{ predicate }}> AS ?p) + <{{ uri }}> ?p ?o . + + OPTIONAL{ + ?o skos:prefLabel ?_label . + } + + BIND(EXISTS{?o rdf:rest ?rest} as ?listItem) + + # This gets set later with the listItemNumber value. + BIND(0 AS ?listItemNumber) + } + GROUP BY ?p ?o ?listItem ?listItemNumber + ORDER BY ?label + LIMIT {{ limit }} + OFFSET {{ offset }} + """ + ).render(uri=uri, predicate=predicate, limit=limit, offset=(page - 1) * limit) + + count = get_predicate_count_index(uri, predicate, sparql_endpoint) + + response = sparql.post(query, sparql_endpoint) + + result = response.json() + + # An index of URIs with label values. + uri_label_index = domain.viewer.resource.json.get_uri_label_index( + result, sparql_endpoint + ) + + # An index of all the URIs linked to and from this resource that are available internally. + uri_internal_index = domain.viewer.resource.json.get_uri_internal_index( + result, sparql_endpoint + ) + + values = [] + + for row in result["results"]["bindings"]: + item = None + + if row["p"]["value"] == str(RDF.type): + continue + else: + if row["o"]["type"] == "uri": + # object_label = uri_label_index.get( + # row["o"]["value"] + # ) or domain.curie.get(row["o"]["value"]) + object_label = ( + uri_label_index.get(row["o"]["value"]) or row["o"]["value"] + ) + item = domain.schema.URI( + label=object_label, + value=row["o"]["value"], + internal=uri_internal_index.get(row["o"]["value"], False), + list_item=True if row["listItem"]["value"] == "true" else False, + list_item_number=row["listItemNumber"]["value"] + if row["listItem"]["value"] == "true" + else None, + ) + elif row["o"]["type"] == "literal": + datatype = row["o"].get("datatype", "") + if datatype: + datatype = domain.schema.URI( + label=datatype, + value=datatype, + internal=uri_internal_index.get(datatype, False), + list_item=True if row["listItem"]["value"] == "true" else False, + list_item_number=row["listItemNumber"]["value"] + if row["listItem"]["value"] == "true" + else None, + ) + else: + datatype = None + + item = domain.schema.Literal( + value=row["o"]["value"], + datatype=datatype, + language=row["o"].get("xml:lang", ""), + list_item=True if row["listItem"]["value"] == "true" else False, + list_item_number=row["listItemNumber"]["value"] + if row["listItem"]["value"] == "true" + else None, + ) + elif row["o"]["type"] == "bnode": + # TODO: Handle blank nodes. + pass + else: + raise ValueError( + f"Expected type to be uri or literal but got {row['o']['type']}" + ) + + if item: + values.append(item) + + predicate_values = PredicateValues( + uri=uri, predicate=predicate, objects=values, count=count + ) + return predicate_values.json() + + +def _get_predicates(uri: str, sparql_endpoint: str) -> list[URI]: + query = Template( + """ + SELECT DISTINCT ?p + WHERE { + <{{ uri }}> ?p ?o . + } + ORDER BY ?p + """ + ).render(uri=uri) + + response = sparql.post(query, sparql_endpoint) + + predicates = [ + URI( + label=domain.curie.get(row["p"]["value"]), + value=row["p"]["value"], + internal=False, + ) + for row in response.json()["results"]["bindings"] + ] + + return predicates + + +def _get_types(uri: str, sparql_endpoint: str) -> list[URI]: + query = Template( + """ + SELECT DISTINCT ?type + WHERE { + <{{ uri }}> a ?type . + FILTER(!isBlank(?type)) + } + ORDER BY ?type + """ + ).render(uri=uri) + + response = sparql.post(query, sparql_endpoint) + + types = [ + URI( + label=domain.label.get(row["type"]["value"], sparql_endpoint) + or domain.curie.get(row["type"]["value"]), + value=row["type"]["value"], + internal=False, + ) + for row in response.json()["results"]["bindings"] + ] + + return types + + +def _exists(uri: str, sparql_endpoint: str) -> bool: + query = Template( + """ + ASK { + <{{ uri }}> ?p ?o . + } + """ + ).render(uri=uri) + + response = sparql.post(query, sparql_endpoint) + + return response.json()["boolean"] + + +def json_renderer(uri: str, sparql_endpoint: str) -> Resource: + if not _exists(uri, sparql_endpoint): + raise SPARQLNotFoundError(f"Resource with URI {uri} not found.") + + label = domain.label.get(uri, sparql_endpoint) + types = _get_types(uri, sparql_endpoint) + predicates = _get_predicates(uri, sparql_endpoint) + predicates = list(filter(lambda x: x.value != str(RDF.type), predicates)) + + return Resource(uri=uri, label=label, types=types, properties=predicates).json() diff --git a/src/linkeddata_api/views/api_v2/viewer/route_entrypoint.py b/src/linkeddata_api/views/api_v2/viewer/route_entrypoint.py new file mode 100644 index 0000000..971418e --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/route_entrypoint.py @@ -0,0 +1,49 @@ +from flask import abort, request +from flask_tern import openapi + +from linkeddata_api.views.api_v2.blueprint import bp +from linkeddata_api import domain +from linkeddata_api.domain.viewer.entrypoints.exceptions import ( + RequestError, + SPARQLResultJSONError, + ViewerIDNotFoundError, +) +from linkeddata_api.domain.pydantic_jsonify import jsonify + + +mapping = { + "nrm": { + "func": domain.viewer.entrypoints.nrm.get, + "sparql_endpoint": "https://graphdb.tern.org.au/repositories/dawe_vocabs_core", + }, + "tern": { + "func": domain.viewer.entrypoints.tern.get, + "sparql_endpoint": "https://graphdb.tern.org.au/repositories/tern_vocabs_core", + }, +} + + +@bp.get("/viewer/entrypoint/") +@openapi.validate(validate_request=False, validate_response=False) +def get_entrypoint(viewer_id: str): + page = int(request.args.get("page", 1)) + if page < 1: + page = 1 + + try: + obj = mapping.get(viewer_id) + if obj is None: + raise ViewerIDNotFoundError(f"Key '{viewer_id}' not found") + + sparql_endpoint = obj["sparql_endpoint"] + entrypoint_items = obj["func"](sparql_endpoint, page) + except ViewerIDNotFoundError as err: + abort(404, str(err)) + except (RequestError, SPARQLResultJSONError) as err: + abort(502, err.description) + except Exception as err: + abort(500, err) + + return jsonify( + entrypoint_items, headers={"cache-control": "max-age=600, s-maxage=3600"} + ) diff --git a/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py b/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py new file mode 100644 index 0000000..7342ecb --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py @@ -0,0 +1,47 @@ +from flask import Response, abort, request +from flask_tern import openapi + +from linkeddata_api.domain.viewer.resource import ( + RequestError, + SPARQLNotFoundError, + SPARQLResultJSONError, +) +from linkeddata_api.views.api_v2.blueprint import bp + +from .json_renderer import get_predicate_values + + +@bp.get("/viewer/predicate-values") +@openapi.validate(validate_request=False, validate_response=False) +def get_resource_predicate_values(): + + uri = request.args.get("uri") + predicate = request.args.get("predicate") + sparql_endpoint = request.args.get("sparql_endpoint") + page = int(request.args.get("page", 1)) + if page < 1: + page = 1 + page_size = int(request.args.get("page_size", 20)) + if page < 1: + page = 20 + + if not uri or not predicate or not sparql_endpoint: + err_msg = "Required query parameters 'uri' or 'predicate' or 'sparql_endpoint' was not provided." + abort(400, err_msg) + + try: + result = get_predicate_values( + uri, predicate, sparql_endpoint, limit=page_size, page=page + ) + except SPARQLNotFoundError as err: + abort(404, err.description) + except (RequestError, SPARQLResultJSONError) as err: + abort(502, err.description) + except Exception as err: + abort(500, err) + + return Response( + result, + mimetype="application/json", + headers={"cache-control": "max-age=600, s-maxage=3600"}, + ) diff --git a/src/linkeddata_api/views/api_v2/viewer/route_resource.py b/src/linkeddata_api/views/api_v2/viewer/route_resource.py new file mode 100644 index 0000000..6abf63c --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/route_resource.py @@ -0,0 +1,57 @@ +from flask import Response, abort, request +from flask_tern import openapi +from pydantic import BaseModel + +from linkeddata_api import domain +from linkeddata_api.domain.viewer.resource import ( + RequestError, + SPARQLNotFoundError, + SPARQLResultJSONError, +) +from linkeddata_api.views.api_v2.blueprint import bp + +from .json_renderer import json_renderer + + +@bp.get("/viewer/resource") +@openapi.validate(validate_request=False, validate_response=False) +def get_resource(): + sparql_endpoint = request.args.get("sparql_endpoint") + uri = request.args.get("uri") + + # TODO: Curently we don't support multiple format types in accept headers. + format_ = request.args.get("format") or request.headers.get("accept") + if not format_ or "," in format_: + format_ = "text/turtle" + + # TODO: We don't support incoming relationshpis for resources yet in the JSON renderer. + include_incoming_relationships = request.args.get("include_incoming_relationships") + include_incoming_relationships = ( + True if include_incoming_relationships == "true" else False + ) + + if not uri or not sparql_endpoint: + err_msg = ( + "Required query parameters 'uri' or 'sparql_endpoint' was not provided." + ) + abort(400, err_msg) + + try: + if format_ == "application/json": + result = json_renderer(uri, sparql_endpoint) + else: + result = domain.viewer.resource.handle_rdf_response( + uri, sparql_endpoint, format_, include_incoming_relationships + ) + except SPARQLNotFoundError as err: + abort(404, err.description) + except (RequestError, SPARQLResultJSONError) as err: + abort(502, err.description) + except Exception as err: + abort(500, err) + + return Response( + result, + mimetype=format_, + headers={"cache-control": "max-age=600, s-maxage=3600"}, + ) diff --git a/src/linkeddata_api/views/api_v2/viewer/schema.py b/src/linkeddata_api/views/api_v2/viewer/schema.py new file mode 100644 index 0000000..376cd60 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/schema.py @@ -0,0 +1,46 @@ +from typing import Union + +from pydantic import BaseModel + + +class RDFListItemMixin(BaseModel): + """An item in an RDF List""" + + list_item: bool = False + list_item_number: int | None = None + + +class URI(RDFListItemMixin): + type: str = "uri" + label: str + value: str + internal: bool + + def __hash__(self): + return hash(self.value) + + +class Literal(RDFListItemMixin): + type: str = "literal" + value: str + datatype: URI = None + language: str = "" + + def __hash__(self): + datatype = self.datatype.value if self.datatype else "" + return hash(self.value + datatype + self.language) + + +class Resource(BaseModel): + uri: str + label: str + profile: Union[str, None] = None + types: list[URI] + properties: list[URI] + + +class PredicateValues(BaseModel): + uri: str + predicate: str + count: int + objects: list[Union[URI, Literal]] diff --git a/src/linkeddata_api/views/home/__init__.py b/src/linkeddata_api/views/home/__init__.py index 4fbca23..80121fe 100644 --- a/src/linkeddata_api/views/home/__init__.py +++ b/src/linkeddata_api/views/home/__init__.py @@ -10,7 +10,7 @@ def home(): The default API is a OpenAPIBlueprint and will redirect to the included swagger UI. """ - return redirect(url_for("api_v1.api_root", _external=True)) + return redirect(url_for("api_v2.api_root", _external=True)) @bp.route("/whoami") From a92ef5fc23cd7e044577d002a83d7d2636ee1428 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 01:58:24 +0000 Subject: [PATCH 2/9] Remove log_time decorator usage --- src/linkeddata_api/data/sparql.py | 2 -- src/linkeddata_api/domain/label.py | 3 --- .../domain/viewer/resource/json/__init__.py | 9 --------- 3 files changed, 14 deletions(-) diff --git a/src/linkeddata_api/data/sparql.py b/src/linkeddata_api/data/sparql.py index 1df6cbe..43596b3 100644 --- a/src/linkeddata_api/data/sparql.py +++ b/src/linkeddata_api/data/sparql.py @@ -1,10 +1,8 @@ import requests from . import exceptions -from linkeddata_api.log_time import log_time -@log_time def post( query: str, sparql_endpoint: str, accept: str = "application/sparql-results+json" ) -> requests.Response: diff --git a/src/linkeddata_api/domain/label.py b/src/linkeddata_api/domain/label.py index e4f3653..543dcd4 100644 --- a/src/linkeddata_api/domain/label.py +++ b/src/linkeddata_api/domain/label.py @@ -3,10 +3,8 @@ from jinja2 import Template from linkeddata_api import data -from linkeddata_api.log_time import log_time -@log_time def get( uri: str, sparql_endpoint: str, @@ -152,7 +150,6 @@ def _get_from_list_query(uris: list[str]) -> str: return query -@log_time def get_from_list( uris: list[str], sparql_endpoint: str, diff --git a/src/linkeddata_api/domain/viewer/resource/json/__init__.py b/src/linkeddata_api/domain/viewer/resource/json/__init__.py index b64f40e..bc3e47e 100644 --- a/src/linkeddata_api/domain/viewer/resource/json/__init__.py +++ b/src/linkeddata_api/domain/viewer/resource/json/__init__.py @@ -11,12 +11,10 @@ from linkeddata_api.domain.viewer.resource.json.sort_property_objects import ( sort_property_objects, ) -from linkeddata_api.log_time import log_time logger = logging.getLogger(__name__) -@log_time def _get_uris_from_rdf_list(uri: str, rows: list, sparql_endpoint: str) -> list[str]: new_uris = [] for row in rows: @@ -44,7 +42,6 @@ def _get_uris_from_rdf_list(uri: str, rows: list, sparql_endpoint: str) -> list[ return new_uris -@log_time def _get_uri_values_and_list_items( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> tuple[list[str], list[str]]: @@ -70,7 +67,6 @@ def _get_uri_values_and_list_items( return uri_values, list_items -@log_time def _add_rows_for_rdf_list_items(result: dict, uri: str, sparql_endpoint: str) -> dict: """Add rdf:List items as new rows to the SPARQL result object @@ -102,7 +98,6 @@ def _add_rows_for_rdf_list_items(result: dict, uri: str, sparql_endpoint: str) - return result -@log_time def get_uri_label_index( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> dict[str, str]: @@ -111,7 +106,6 @@ def get_uri_label_index( return uri_label_index -@log_time def get_uri_internal_index( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> dict[str, str]: @@ -122,7 +116,6 @@ def get_uri_internal_index( return uri_internal_index -@log_time def get(uri: str, sparql_endpoint: str) -> domain.schema.Resource: query = f""" SELECT ?p ?o ?listItem ?listItemNumber @@ -168,7 +161,6 @@ def get(uri: str, sparql_endpoint: str) -> domain.schema.Resource: ) -@log_time def _get_incoming_properties(uri: str, sparql_endpoint: str): query = f""" SELECT ?p ?o ?listItem ?listItemNumber @@ -231,7 +223,6 @@ def _get_incoming_properties(uri: str, sparql_endpoint: str): return incoming_properties -@log_time def get_types_and_properties( result: dict, sparql_endpoint: str, uri: Optional[str] = None ) -> tuple[list[domain.schema.URI], list[domain.schema.PredicateObjects]]: From d4a6aaa8a1512a85631f5b885e5f896fda4a54ee Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 01:59:16 +0000 Subject: [PATCH 3/9] Add custom profile Method and MethodCollection to new paginated json renderer --- .../views/api_v2/viewer/json_renderer.py | 187 ++++++++++++++---- .../views/api_v2/viewer/profile/__init__.py | 0 .../api_v2/viewer/profile/base_profile.py | 67 +++++++ .../api_v2/viewer/profile/custom_profiles.py | 182 +++++++++++++++++ .../api_v2/viewer/profile/registration.py | 12 ++ .../api_v2/viewer/route_predicate_values.py | 3 +- .../views/api_v2/viewer/schema.py | 1 + 7 files changed, 416 insertions(+), 36 deletions(-) create mode 100644 src/linkeddata_api/views/api_v2/viewer/profile/__init__.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/profile/base_profile.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py create mode 100644 src/linkeddata_api/views/api_v2/viewer/profile/registration.py diff --git a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py index 6634460..be4f78e 100644 --- a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py +++ b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py @@ -1,3 +1,4 @@ +from typing import Optional from jinja2 import Template from rdflib import RDF @@ -6,53 +7,146 @@ from linkeddata_api.data.exceptions import SPARQLNotFoundError from .schema import URI, Resource, PredicateValues +from .profile.base_profile import get_profile -def get_predicate_count_index(uri: str, predicate: str, sparql_endpoint: str) -> int: +def predicate_is_list_item(uri: str, predicate: str, sparql_endpoint: str) -> bool: query = Template( """ - SELECT (COUNT(DISTINCT(?value)) as ?count) - WHERE { - <{{ uri }}> <{{ predicate }}> ?value . + PREFIX rdf: + ASK { + <{{ uri }}> <{{ predicate }}> ?o . + ?o rdf:rest ?rest } - """ + """ ).render(uri=uri, predicate=predicate) response = sparql.post(query, sparql_endpoint) + data = response.json() + + return data["boolean"] + + +def get_predicate_count_index( + uri: str, predicate: str, sparql_endpoint: str, profile: str +) -> int: + ProfileClass = get_profile(profile) + + if ProfileClass: + profile_instance = ProfileClass(uri, []) + count = profile_instance.get_predicate_values_count(predicate, sparql_endpoint) + + else: + is_list_item = predicate_is_list_item(uri, predicate, sparql_endpoint) + + if is_list_item: + query = Template( + """ + PREFIX rdf: + SELECT (COUNT(DISTINCT(?value)) as ?count) + WHERE { + <{{ uri }}> <{{ predicate }}> ?o . + ?o rdf:rest* ?rest . + ?rest rdf:first ?value + } + """ + ).render(uri=uri, predicate=predicate) + else: + query = Template( + """ + SELECT (COUNT(DISTINCT(?value)) as ?count) + WHERE { + <{{ uri }}> <{{ predicate }}> ?value . + } + """ + ).render(uri=uri, predicate=predicate) + + response = sparql.post(query, sparql_endpoint) + + count = int(response.json()["results"]["bindings"][0]["count"]["value"]) - count = int(response.json()["results"]["bindings"][0]["count"]["value"]) return count +def get_predicate_values_query( + uri: str, + predicate: str, + sparql_endpoint: str, + limit: int, + page: int, + profile: str = "", +) -> str: + is_list_item = predicate_is_list_item(uri, predicate, sparql_endpoint) + + ProfileClass = get_profile(profile) + + if ProfileClass: + profile_instance = ProfileClass(uri, []) + query = profile_instance.get_predicate_values(predicate, limit, page) + else: + if is_list_item: + query = Template( + """ + PREFIX rdf: + PREFIX skos: + SELECT ?p ?o ?listItem ?listItemNumber + WHERE { + BIND(<{{ predicate }}> AS ?p) + <{{ uri }}> ?p ?_o . + ?_o rdf:rest* ?rest . + ?rest rdf:first ?o + + BIND(EXISTS{?o rdf:rest ?rest} as ?listItem) + + # This gets set later with the listItemNumber value. + BIND(0 AS ?listItemNumber) + } + GROUP BY ?p ?o ?listItem ?listItemNumber + LIMIT {{ limit }} + OFFSET {{ offset }} + """ + ).render( + uri=uri, predicate=predicate, limit=limit, offset=(page - 1) * limit + ) + else: + query = Template( + """ + PREFIX rdf: + PREFIX skos: + SELECT ?p ?o ?listItem ?listItemNumber (SAMPLE(?_label) AS ?label) + WHERE { + BIND(<{{ predicate }}> AS ?p) + <{{ uri }}> ?p ?o . + + OPTIONAL{ + ?o skos:prefLabel ?_label . + } + + BIND(EXISTS{?o rdf:rest ?rest} as ?listItem) + + # This gets set later with the listItemNumber value. + BIND(0 AS ?listItemNumber) + } + GROUP BY ?p ?o ?listItem ?listItemNumber + ORDER BY ?label + LIMIT {{ limit }} + OFFSET {{ offset }} + """ + ).render( + uri=uri, predicate=predicate, limit=limit, offset=(page - 1) * limit + ) + + return query + + def get_predicate_values( - uri: str, predicate: str, sparql_endpoint: str, limit: int, page: int + uri: str, predicate: str, sparql_endpoint: str, profile: str, limit: int, page: int ) -> PredicateValues: - query = Template( - """ - PREFIX rdf: - PREFIX skos: - SELECT ?p ?o ?listItem ?listItemNumber (SAMPLE(?_label) AS ?label) - WHERE { - BIND(<{{ predicate }}> AS ?p) - <{{ uri }}> ?p ?o . - - OPTIONAL{ - ?o skos:prefLabel ?_label . - } - - BIND(EXISTS{?o rdf:rest ?rest} as ?listItem) - - # This gets set later with the listItemNumber value. - BIND(0 AS ?listItemNumber) - } - GROUP BY ?p ?o ?listItem ?listItemNumber - ORDER BY ?label - LIMIT {{ limit }} - OFFSET {{ offset }} - """ - ).render(uri=uri, predicate=predicate, limit=limit, offset=(page - 1) * limit) - count = get_predicate_count_index(uri, predicate, sparql_endpoint) + count = get_predicate_count_index(uri, predicate, sparql_endpoint, profile) + query = get_predicate_values_query( + uri, predicate, sparql_endpoint, limit, page, profile + ) response = sparql.post(query, sparql_endpoint) @@ -133,7 +227,7 @@ def get_predicate_values( return predicate_values.json() -def _get_predicates(uri: str, sparql_endpoint: str) -> list[URI]: +def get_predicates(uri: str, sparql_endpoint: str) -> list[URI]: query = Template( """ SELECT DISTINCT ?p @@ -205,7 +299,30 @@ def json_renderer(uri: str, sparql_endpoint: str) -> Resource: label = domain.label.get(uri, sparql_endpoint) types = _get_types(uri, sparql_endpoint) - predicates = _get_predicates(uri, sparql_endpoint) + predicates = get_predicates(uri, sparql_endpoint) predicates = list(filter(lambda x: x.value != str(RDF.type), predicates)) - return Resource(uri=uri, label=label, types=types, properties=predicates).json() + profile_uri = "" + ProfileClass = None + for t in types: + ProfileClass = get_profile(t.value) + + if ProfileClass: + break + + if ProfileClass: + profile = ProfileClass(uri, predicates) + profile.add_and_remove() + predicates = profile.properties + profile_uri = profile.uri + + return Resource( + uri=uri, + label=label, + types=types, + properties=predicates, + profile=profile_uri, + properties_require_profile=profile.properties_require_profile + if ProfileClass + else [], + ).json() diff --git a/src/linkeddata_api/views/api_v2/viewer/profile/__init__.py b/src/linkeddata_api/views/api_v2/viewer/profile/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/linkeddata_api/views/api_v2/viewer/profile/base_profile.py b/src/linkeddata_api/views/api_v2/viewer/profile/base_profile.py new file mode 100644 index 0000000..60e92ce --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/profile/base_profile.py @@ -0,0 +1,67 @@ +from typing import Union +from abc import ABCMeta, abstractmethod + +from ..schema import URI + + +profiles = {} + + +class Profile(metaclass=ABCMeta): + resource_uri: str + properties: list[URI] + properties_require_profile: list[str] = [] + + def __init__(self, resource_uri: str, properties: list[URI]) -> None: + self.resource_uri = resource_uri + self.properties = properties + + @staticmethod + def _add_and_remove_property( + predicate_uri: str, + old_list: list[URI], + new_list: list[URI], + ) -> Union[URI, None]: + """Add and remove the PredicateObjects object if matched by predicate_uri in + the referenced lists, 'old_list' and 'new_list' + + Returns a copy of the PredicateObjects object, if found, else None. + + Use linkeddata_api.domain.curie.get() to get the label of predicates. + Use linkeddata_api.domain.label.get_from_list() to get a dict of labels for the values. + """ + for uri in old_list: + if uri.value == predicate_uri: + new_list.append(uri) + + old_list.remove(uri) + return uri + + @property + def uri(self) -> str: + """Get the URI of the RDF class this profile targets""" + return self._uri() + + @abstractmethod + def _uri(self) -> str: + ... + + @abstractmethod + def add_and_remove(self): + ... + + +def register_profile(uri: str, profile_class: type[Profile]) -> None: + profiles.update({uri: profile_class}) + + +def get_profile(uri) -> type[Profile] | None: + try: + profile = profiles[uri] + return profile + except KeyError: + pass + + +# Register the profiles with an import statement +from . import registration diff --git a/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py b/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py new file mode 100644 index 0000000..7d432cb --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py @@ -0,0 +1,182 @@ +from jinja2 import Template +from rdflib import RDFS, SKOS, SDO, DCTERMS + +from linkeddata_api.data import sparql +from linkeddata_api.domain.namespaces import TERN + +from .base_profile import Profile +from ..schema import PredicateValues, URI + + +class MethodCollectionProfile(Profile): + def _uri(self) -> str: + return "https://w3id.org/tern/ontologies/tern/MethodCollection" + + def add_and_remove(self): + super().add_and_remove() + + properties = self.properties + new_properties = [] + + self._add_and_remove_property(str(RDFS.isDefinedBy), properties, new_properties) + + self._add_and_remove_property(str(SKOS.prefLabel), properties, new_properties) + # Pop to omit skos:prefLabel property + new_properties.pop() + + self._add_and_remove_property(str(TERN), properties, new_properties) + self._add_and_remove_property(str(SDO.url), properties, new_properties) + self._add_and_remove_property(str(SKOS.memberList), properties, new_properties) + self._add_and_remove_property(str(TERN.scope), properties, new_properties) + self._add_and_remove_property(str(SKOS.definition), properties, new_properties) + self._add_and_remove_property(str(TERN.purpose), properties, new_properties) + self._add_and_remove_property( + str(DCTERMS.description), properties, new_properties + ) + self._add_and_remove_property(str(TERN.equipment), properties, new_properties) + self._add_and_remove_property( + str(TERN.instructions), properties, new_properties + ) + self._add_and_remove_property(str(SKOS.note), properties, new_properties) + self._add_and_remove_property(str(DCTERMS.source), properties, new_properties) + self._add_and_remove_property(str(TERN.appendix), properties, new_properties) + + self.properties = new_properties + properties + + +class MethodProfile(MethodCollectionProfile): + properties_require_profile = [ + "https://w3id.org/tern/ontologies/tern/hasObservableProperty", + "https://w3id.org/tern/ontologies/tern/hasFeatureType", + "https://w3id.org/tern/ontologies/tern/hasCategoricalValuesCollection", + ] + + def _uri(self) -> str: + return "https://w3id.org/tern/ontologies/tern/Method" + + def get_additional_values( + self, + metadata_predicate: str, + predicate: str, + limit: int, + page: int, + ): + query = Template( + """ + PREFIX skos: + PREFIX tern: + select ?p ?o ?listItem ?listItemNumber (SAMPLE(?_label) AS ?label) + where { + ?_observable_property_meta <{{ uri }}> ; + <{{ metadata_predicate }}> ?_object_value . + BIND(<{{ predicate }}> AS ?p) + BIND(?_object_value AS ?o) + BIND(false AS ?listItem) + + OPTIONAL{ + ?o skos:prefLabel ?_label . + } + + # This gets set later with the listItemNumber value. + BIND(0 AS ?listItemNumber) + } + GROUP BY ?p ?o ?listItem ?listItemNumber + ORDER BY ?label + LIMIT {{ limit }} + OFFSET {{ offset }} + """ + ).render( + uri=self.resource_uri, + metadata_predicate=metadata_predicate, + predicate=predicate, + limit=limit, + offset=(page - 1) * limit, + ) + + return query + + def _get_predicate_values_count( + self, metadata_predicate: str, sparql_endpoint: str + ) -> int: + query = Template( + """ + SELECT (COUNT(DISTINCT(?value)) as ?count) + WHERE { + ?_observable_property_meta <{{ uri }}> ; + <{{ metadata_predicate }}> ?value . + } + """ + ).render(uri=self.resource_uri, metadata_predicate=metadata_predicate) + + response = sparql.post(query, sparql_endpoint) + + count = int(response.json()["results"]["bindings"][0]["count"]["value"]) + + return count + + def get_predicate_values_count(self, predicate: str, sparql_endpoint: str) -> int: + if predicate == "https://w3id.org/tern/ontologies/tern/hasObservableProperty": + return self._get_predicate_values_count( + "urn:property:observableProperty", sparql_endpoint + ) + if predicate == "https://w3id.org/tern/ontologies/tern/hasFeatureType": + return self._get_predicate_values_count( + "urn:property:featureType", sparql_endpoint + ) + if ( + predicate + == "https://w3id.org/tern/ontologies/tern/hasCategoricalValuesCollection" + ): + return self._get_predicate_values_count( + "urn:property:categoricalValuesCollection", sparql_endpoint + ) + + def get_predicate_values( + self, predicate: str, limit: int, page: int + ) -> list[PredicateValues]: + + if predicate == "https://w3id.org/tern/ontologies/tern/hasObservableProperty": + return self.get_additional_values( + "urn:property:observableProperty", + "https://w3id.org/tern/ontologies/tern/hasObservableProperty", + limit, + page, + ) + if predicate == "https://w3id.org/tern/ontologies/tern/hasFeatureType": + return self.get_additional_values( + "urn:property:featureType", + "https://w3id.org/tern/ontologies/tern/hasFeatureType", + limit, + page, + ) + if ( + predicate + == "https://w3id.org/tern/ontologies/tern/hasCategoricalValuesCollection" + ): + return self.get_additional_values( + "urn:property:categoricalValuesCollection", + "https://w3id.org/tern/ontologies/tern/hasCategoricalValuesCollection", + limit, + page, + ) + + def add_and_remove(self): + super().add_and_remove() + + self.properties += [ + URI( + label="tern:hasObservableProperty", + value="https://w3id.org/tern/ontologies/tern/hasObservableProperty", + internal=False, + ), + URI( + label="tern:hasFeatureType", + value="https://w3id.org/tern/ontologies/tern/hasFeatureType", + internal=False, + ), + URI( + label="tern:hasCategoricalValuesCollection", + value="https://w3id.org/tern/ontologies/tern/hasCategoricalValuesCollection", + internal=False, + ), + ] diff --git a/src/linkeddata_api/views/api_v2/viewer/profile/registration.py b/src/linkeddata_api/views/api_v2/viewer/profile/registration.py new file mode 100644 index 0000000..54ce096 --- /dev/null +++ b/src/linkeddata_api/views/api_v2/viewer/profile/registration.py @@ -0,0 +1,12 @@ +from .base_profile import register_profile +from .custom_profiles import MethodCollectionProfile, MethodProfile + + +register_profile( + "https://w3id.org/tern/ontologies/tern/MethodCollection", + MethodCollectionProfile, +) +register_profile( + "https://w3id.org/tern/ontologies/tern/Method", + MethodProfile, +) diff --git a/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py b/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py index 7342ecb..edd45c6 100644 --- a/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py +++ b/src/linkeddata_api/views/api_v2/viewer/route_predicate_values.py @@ -18,6 +18,7 @@ def get_resource_predicate_values(): uri = request.args.get("uri") predicate = request.args.get("predicate") sparql_endpoint = request.args.get("sparql_endpoint") + profile = request.args.get("profile", "") page = int(request.args.get("page", 1)) if page < 1: page = 1 @@ -31,7 +32,7 @@ def get_resource_predicate_values(): try: result = get_predicate_values( - uri, predicate, sparql_endpoint, limit=page_size, page=page + uri, predicate, sparql_endpoint, profile, limit=page_size, page=page ) except SPARQLNotFoundError as err: abort(404, err.description) diff --git a/src/linkeddata_api/views/api_v2/viewer/schema.py b/src/linkeddata_api/views/api_v2/viewer/schema.py index 376cd60..12c4ae0 100644 --- a/src/linkeddata_api/views/api_v2/viewer/schema.py +++ b/src/linkeddata_api/views/api_v2/viewer/schema.py @@ -37,6 +37,7 @@ class Resource(BaseModel): profile: Union[str, None] = None types: list[URI] properties: list[URI] + properties_require_profile: list[str] class PredicateValues(BaseModel): From affa14ea0abbf9e605baa48f95faec426d16cc55 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 01:59:42 +0000 Subject: [PATCH 4/9] Remove unused import --- src/linkeddata_api/views/api_v2/viewer/json_renderer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py index be4f78e..649bdb5 100644 --- a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py +++ b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py @@ -1,4 +1,3 @@ -from typing import Optional from jinja2 import Template from rdflib import RDF From 194aa6af57acef1cab51f12b3c61f6a303ce62ab Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 02:00:19 +0000 Subject: [PATCH 5/9] Remove unused import --- src/linkeddata_api/views/api_v2/viewer/route_resource.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/linkeddata_api/views/api_v2/viewer/route_resource.py b/src/linkeddata_api/views/api_v2/viewer/route_resource.py index 6abf63c..1bd60d9 100644 --- a/src/linkeddata_api/views/api_v2/viewer/route_resource.py +++ b/src/linkeddata_api/views/api_v2/viewer/route_resource.py @@ -1,6 +1,5 @@ from flask import Response, abort, request from flask_tern import openapi -from pydantic import BaseModel from linkeddata_api import domain from linkeddata_api.domain.viewer.resource import ( From 27f86f362d347ed88b55a788f0de6d607c84b309 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 02:06:35 +0000 Subject: [PATCH 6/9] Update test to point to v2 --- tests/test_home.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_home.py b/tests/test_home.py index 02a26f2..2788076 100644 --- a/tests/test_home.py +++ b/tests/test_home.py @@ -1,7 +1,5 @@ import base64 -import pytest - def test_root(client): response = client.get("/") @@ -14,7 +12,7 @@ def test_home(client): assert response.headers["Location"] == "http://localhost/api/" response = client.get("/api/") - assert response.headers["Location"] == "http://localhost/api/v1.0/" + assert response.headers["Location"] == "http://localhost/api/v2.0/" def test_whoami_fail(client): From b353ced427386bf43f7527947ecc116f48944d20 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 02:18:23 +0000 Subject: [PATCH 7/9] Remove commented out code --- src/linkeddata_api/views/api_v2/viewer/json_renderer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py index 649bdb5..2047899 100644 --- a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py +++ b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py @@ -170,9 +170,6 @@ def get_predicate_values( continue else: if row["o"]["type"] == "uri": - # object_label = uri_label_index.get( - # row["o"]["value"] - # ) or domain.curie.get(row["o"]["value"]) object_label = ( uri_label_index.get(row["o"]["value"]) or row["o"]["value"] ) From 71fad2dddacc23c853078bd113743fa03d4d8895 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 04:17:23 +0000 Subject: [PATCH 8/9] Remove line that is doing nothing --- .../views/api_v2/viewer/profile/custom_profiles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py b/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py index 7d432cb..f7a3fe5 100644 --- a/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py +++ b/src/linkeddata_api/views/api_v2/viewer/profile/custom_profiles.py @@ -24,7 +24,6 @@ def add_and_remove(self): # Pop to omit skos:prefLabel property new_properties.pop() - self._add_and_remove_property(str(TERN), properties, new_properties) self._add_and_remove_property(str(SDO.url), properties, new_properties) self._add_and_remove_property(str(SKOS.memberList), properties, new_properties) self._add_and_remove_property(str(TERN.scope), properties, new_properties) From 008fbdd2c2974246867239af9f291fb10a54a591 Mon Sep 17 00:00:00 2001 From: Edmond Chuc Date: Wed, 25 Jan 2023 04:55:51 +0000 Subject: [PATCH 9/9] Return the uri as label if label not found --- src/linkeddata_api/views/api_v2/viewer/json_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py index 2047899..6f3e89f 100644 --- a/src/linkeddata_api/views/api_v2/viewer/json_renderer.py +++ b/src/linkeddata_api/views/api_v2/viewer/json_renderer.py @@ -314,7 +314,7 @@ def json_renderer(uri: str, sparql_endpoint: str) -> Resource: return Resource( uri=uri, - label=label, + label=label or uri, types=types, properties=predicates, profile=profile_uri,