Wikidata <> authors integration: first steps

openlibrary/core/models.py

@@ -28,6 +28,7 @@
 from openlibrary.core.ratings import Ratings
 from openlibrary.utils import extract_numeric_id_from_olid, dateutil
 from openlibrary.utils.isbn import to_isbn_13, isbn_13_to_isbn_10, canonical
+from openlibrary.core.wikidata import WikidataEntity, get_wikidata_entity
 
 from . import cache, waitinglist
 
@@ -756,6 +757,15 @@ def url(self, suffix="", **params):
     def get_url_suffix(self):
         return self.name or "unnamed"
 
+    def wikidata(
+        self, bust_cache: bool = False, fetch_missing: bool = False
+    ) -> WikidataEntity | None:
+        if wd_id := self.remote_ids.get("wikidata"):
+            return get_wikidata_entity(
+                qid=wd_id, bust_cache=bust_cache, fetch_missing=fetch_missing
+            )
+        return None
+
     def __repr__(self):
         return "<Author: %s>" % repr(self.key)
 

openlibrary/core/schema.sql

@@ -90,3 +90,9 @@ CREATE TABLE yearly_reading_goals (
     updated timestamp without time zone default (current_timestamp at time zone 'utc'),
     primary key (username, year)
 );
+
+CREATE TABLE wikidata (
+    id text not null primary key,
+    data json,
+    updated timestamp without time zone default (current_timestamp at time zone 'utc')
+)

openlibrary/core/wikidata.py

@@ -0,0 +1,145 @@
+"""
+The purpose of this file is to:
+1. Interact with the Wikidata API
+2. Store the results
+3. Make the results easy to access from other files
+"""
+
+import requests
+import logging
+from dataclasses import dataclass
+from openlibrary.core.helpers import days_since
+
+from datetime import datetime
+import json
+from openlibrary.core import db
+
+logger = logging.getLogger("core.wikidata")
+
+WIKIDATA_API_URL = 'https://www.wikidata.org/w/rest.php/wikibase/v0/entities/items/'
+WIKIDATA_CACHE_TTL_DAYS = 30
+
+
+@dataclass
+class WikidataEntity:
+    """
+    This is the model of the api response from WikiData plus the updated field
+    https://www.wikidata.org/wiki/Wikidata:REST_API
+    """
+
+    id: str
+    type: str
+    labels: dict[str, str]
+    descriptions: dict[str, str]
+    aliases: dict[str, list[str]]
+    statements: dict[str, dict]
+    sitelinks: dict[str, dict]
+    _updated: datetime  # This is when we fetched the data, not when the entity was changed in Wikidata
+
+    def get_description(self, language: str = 'en') -> str | None:
+        """If a description isn't available in the requested language default to English"""
+        return self.descriptions.get(language) or self.descriptions.get('en')
+
+    @classmethod
+    def from_dict(cls, response: dict, updated: datetime):
+        return cls(
+            **response,
+            _updated=updated,
+        )
+
+    def to_wikidata_api_json_format(self) -> str:
+        """
+        Transforms the dataclass a JSON string like we get from the Wikidata API.
+        This is used for storing the json in the database.
+        """
+        entity_dict = {
+            'id': self.id,
+            'type': self.type,
+            'labels': self.labels,
+            'descriptions': self.descriptions,
+            'aliases': self.aliases,
+            'statements': self.statements,
+            'sitelinks': self.sitelinks,
+        }
+        return json.dumps(entity_dict)
+
+
+def _cache_expired(entity: WikidataEntity) -> bool:
+    return days_since(entity._updated) > WIKIDATA_CACHE_TTL_DAYS
+
+
+def get_wikidata_entity(
+    qid: str, bust_cache: bool = False, fetch_missing: bool = False
+) -> WikidataEntity | None:
+    """
+    This only supports QIDs, if we want to support PIDs we need to use different endpoints
+    By default this will only use the cache (unless it is expired).
+    This is to avoid overwhelming Wikidata servers with requests from every visit to an author page.
+    bust_cache must be set to True if you want to fetch new items from Wikidata.
+    # TODO: After bulk data imports we should set fetch_missing to true (or remove it).
+    """
+    if bust_cache:
+        _get_from_web(qid)
+
+    if entity := _get_from_cache(qid):
+        if _cache_expired(entity):
+            return _get_from_web(qid)
+        return entity
+
+    if fetch_missing and not entity:
+        return _get_from_web(qid)
+
+    return None
+
+
+def _get_from_web(id: str) -> WikidataEntity | None:
+    response = requests.get(f'{WIKIDATA_API_URL}{id}')
+    if response.status_code == 200:
+        entity = WikidataEntity.from_dict(
+            response=response.json(), updated=datetime.now()
+        )
+        _add_to_cache(entity)
+        return entity
+    else:
+        logger.error(f'Wikidata Response: {response.status_code}, id: {id}')
+        return None
+    # Responses documented here https://doc.wikimedia.org/Wikibase/master/js/rest-api/
+
+
+def _get_from_cache_by_ids(ids: list[str]) -> list[WikidataEntity]:
+    response = list(
+        db.get_db().query(
+            'select * from wikidata where id IN ($ids)',
+            vars={'ids': ids},
+        )
+    )
+    return [
+        WikidataEntity.from_dict(response=r.data, updated=r.updated) for r in response
+    ]
+
+
+def _get_from_cache(id: str) -> WikidataEntity | None:
+    """
+    The cache is OpenLibrary's Postgres instead of calling the Wikidata API
+    """
+    if result := _get_from_cache_by_ids([id]):
+        return result[0]
+    return None
+
+
+def _add_to_cache(entity: WikidataEntity) -> None:
+    # TODO: after we upgrade to postgres 9.5+ we should use upsert here
+    oldb = db.get_db()
+    json_data = entity.to_wikidata_api_json_format()
+
+    if _get_from_cache(entity.id):
+        return oldb.update(
+            "wikidata",
+            where="id=$id",
+            vars={'id': entity.id},
+            data=json_data,
+            updated=entity._updated,
+        )
+    else:
+        # We don't provide the updated column on insert because postgres defaults to the current time
+        return oldb.insert("wikidata", id=entity.id, data=json_data)

openlibrary/plugins/wikidata/__init__.py

@@ -0,0 +1 @@
+'wikidata plugin.'

openlibrary/templates/type/author/edit.html

@@ -35,6 +35,11 @@ <h1>$_("Edit Author")</h1>
                 <input type="text" name="author--name" id="name" value="$page.name" class="required"/>
             </div>
         </div>
+        <p style="text-align: center;">
+            $ wikidata = page.wikidata(bust_cache=True, fetch_missing=True)
+            $if wikidata:
+                $wikidata.get_description(i18n.get_locale())
+        </p>
     </div>
 
         <div>

openlibrary/templates/type/author/view.html

@@ -170,6 +170,16 @@ <h6 class="collapse black uppercase">$label</h6>
             $:render_subjects(_("Time"), books.facet_counts.get('time_facet'), 'time:')
         <!-- /SUBJECTS -->
 
+        <div class="section">
+            <h6 class="collapse black uppercase">TESTING ONLY WIKIDATA SECTION</h6>
+            <p style="text-align: center;">
+                <!-- This is only below subject temporarily to avoid merge conflicts on testing -->
+                $ wikidata = page.wikidata(fetch_missing=show_librarian_extras)
+                $if wikidata:
+                    $wikidata.get_description(i18n.get_locale())
+            </p>
+        </div>
+
         $if "lists" in ctx.features:
             <div class="section Tools">
                 $:render_template("lists/widget", page, include_rating=False, exclude_own_lists=True, show_active_lists=True)