Wikidata <> authors integration: first steps

openlibrary/core/models.py

@@ -28,6 +28,7 @@
 from openlibrary.core.ratings import Ratings
 from openlibrary.utils import extract_numeric_id_from_olid, dateutil
 from openlibrary.utils.isbn import to_isbn_13, isbn_13_to_isbn_10, canonical
+from openlibrary.core.wikidata import WikidataEntity, get_wikidata_entity
 
 from . import cache, waitinglist
 
@@ -756,6 +757,15 @@ def url(self, suffix="", **params):
     def get_url_suffix(self):
         return self.name or "unnamed"
 
+    def wikidata(
+        self, bust_cache: bool = False, fetch_missing: bool = False
+    ) -> WikidataEntity | None:
+        if wd_id := self.remote_ids.get("wikidata"):
+            return get_wikidata_entity(
+                QID=wd_id, bust_cache=bust_cache, fetch_missing=fetch_missing
+            )
+        return None
+
     def __repr__(self):
         return "<Author: %s>" % repr(self.key)
 

openlibrary/core/schema.sql

@@ -90,3 +90,9 @@ CREATE TABLE yearly_reading_goals (
     updated timestamp without time zone default (current_timestamp at time zone 'utc'),
     primary key (username, year)
 );
+
+CREATE TABLE wikidata (
+    id text not null primary key,
+    data json,
+    updated timestamp without time zone default (current_timestamp at time zone 'utc')
+)

openlibrary/core/wikidata.py

@@ -0,0 +1,147 @@
+"""
+The purpose of this file is to:
+1. Interact with the Wikidata API
+2. Store the results
+3. Make the results easy to access from other files
+"""
+
+import requests
+from dataclasses import dataclass
+from openlibrary.core.helpers import days_since
+
+from datetime import datetime
+import json
+from openlibrary.core import db
+
+WIKIDATA_API_URL = 'https://www.wikidata.org/w/rest.php/wikibase/v0/entities/items/'
+WIKIDATA_CACHE_TTL_DAYS = 30
+
+
+@dataclass
+class WikidataEntity:
+    """
+    This is the model of the api response from WikiData plus the updated field
+    https://www.wikidata.org/wiki/Wikidata:REST_API
+    """
+
+    id: str
+    type: str
+    labels: dict
+    descriptions: dict
+    aliases: dict
+    statements: dict
+    sitelinks: dict
+    updated: datetime  # This is when we fetched the data, not when the entity was changed in Wikidata
+
+    def description(self, language: str = 'en') -> str | None:
+        """If a description isn't available in the requested language default to English"""
+        return self.descriptions.get(language) or self.descriptions.get('en')
+
+    @classmethod
+    def from_dict(cls, response: dict, updated: datetime):
+        return cls(
+            id=response['id'],
+            type=response['type'],
+            labels=response['labels'],
+            descriptions=response['descriptions'],
+            aliases=response['aliases'],
+            statements=response['statements'],
+            sitelinks=response['sitelinks'],
+            updated=updated,
+        )
+
+    def as_api_response_str(self) -> str:
+        """
+        Transforms the dataclass a JSON string like we get from the Wikidata API.
+        This is used for staring the json in the database.
+        """
+        entity_dict = {
+            'id': self.id,
+            'type': self.type,
+            'labels': self.labels,
+            'descriptions': self.descriptions,
+            'aliases': self.aliases,
+            'statements': self.statements,
+            'sitelinks': self.sitelinks,
+        }
+        return json.dumps(entity_dict)
+
+
+def _cache_expired(entity: WikidataEntity) -> bool:
+    return days_since(entity.updated) > WIKIDATA_CACHE_TTL_DAYS
+
+
+def get_wikidata_entity(
+    QID: str, bust_cache: bool = False, fetch_missing: bool = False
+) -> WikidataEntity | None:
+    """
+    This only supports QIDs, if we want to support PIDs we need to use different endpoints
+    By default this will only use the cache (unless it is expired).
+    This is to avoid overwhelming Wikidata servers with requests from every visit to an author page.
+    bust_cache must be set to True if you want to fetch new items from Wikidata.
+    """
+    if bust_cache:
+        _get_from_web(QID)
+
+    if entity := _get_from_cache(QID):
+        if _cache_expired(entity):
+            return _get_from_web(QID)
+        return entity
+
+    if fetch_missing and not entity:
+        return _get_from_web(QID)
+
+    return None
+
+
+def _get_from_web(id: str) -> WikidataEntity | None:
+    response = requests.get(f'{WIKIDATA_API_URL}{id}')
+    if response.status_code == 200:
+        entity = WikidataEntity.from_dict(
+            response=response.json(), updated=datetime.now()
+        )
+        _add_to_cache(entity)
+        return entity
+    else:
+        return None
+    # TODO: What should we do in non-200 cases?
+    # They're documented here https://doc.wikimedia.org/Wikibase/master/js/rest-api/
+
+
+def _get_from_cache_by_ids(ids: list[str]) -> list[WikidataEntity]:
+    response = list(
+        db.get_db().query(
+            'select * from wikidata where id IN ($ids)',
+            vars={'ids': ids},
+        )
+    )
+    return [
+        WikidataEntity.from_dict(response=r.data, updated=r.updated) for r in response
+    ]
+
+
+def _get_from_cache(id: str) -> WikidataEntity | None:
+    """
+    The cache is OpenLibrary's Postgres instead of calling the Wikidata API
+    """
+    if len(result := _get_from_cache_by_ids([id])) > 0:
+        return result[0]
+    return None
+
+
+def _add_to_cache(entity: WikidataEntity) -> None:
+    # TODO: after we upgrade to postgres 9.5+ we should use upsert here
+    oldb = db.get_db()
+    json_data = entity.as_api_response_str()
+
+    if _get_from_cache(entity.id):
+        return oldb.update(
+            "wikidata",
+            where="id=$id",
+            vars={'id': entity.id},
+            data=json_data,
+            updated=entity.updated,
+        )
+    else:
+        # We don't provide the updated column on insert because postgres defaults to the current time
+        return oldb.insert("wikidata", id=entity.id, data=json_data)

openlibrary/plugins/wikidata/__init__.py

@@ -0,0 +1 @@
+'wikidata plugin.'

openlibrary/templates/type/author/edit.html

@@ -21,7 +21,6 @@ <h1>$_("Edit Author")</h1>
 
     <form id="addAuthor" name="edit" method="post" action="" class="olform books validate">
 
-
     <div class="formElement title">
         <div class="formElement">
             <div class="label">
@@ -35,6 +34,9 @@ <h1>$_("Edit Author")</h1>
                 <input type="text" name="author--name" id="name" value="$page.name" class="required"/>
             </div>
         </div>
+        <p style="text-align: center;">
+            $page.wikidata(bust_cache=True, fetch_missing=True).description(i18n.get_locale())
+        </p>
     </div>
 
         <div>

openlibrary/templates/type/author/view.html

@@ -153,6 +153,9 @@ <h2 class="collapse">
             $:render_template("covers/author_photo", page)
             $:render_template("covers/change", page, ".bookCover img")
         </div>
+        <p style="text-align: center;">
+            $page.wikidata(fetch_missing=show_librarian_extras).description(i18n.get_locale())
+        </p>
 
         $def render_subjects(label, subjects, prefix):
             $if subjects: