diff --git a/backend/addcorpus/python_corpora/corpus.py b/backend/addcorpus/python_corpora/corpus.py index 220a427bc..3b0e2594a 100644 --- a/backend/addcorpus/python_corpora/corpus.py +++ b/backend/addcorpus/python_corpora/corpus.py @@ -79,23 +79,6 @@ def category(self): ''' raise NotImplementedError('CorpusDefinition missing category') - ''' - Directory where source data is located - If neither `data_directory` nor `data_url` is set to valid paths, this corpus cannot be indexed - ''' - data_directory = None - - ''' - URL where source data is located - If neither `data_directory` nor `data_url` is set to valid paths, this corpus cannot be indexed - ''' - data_url = None - - ''' - If connecting to the data URL requires and API key, it needs to be set here - ''' - data_api_key = None - @property def es_index(self): ''' diff --git a/backend/addcorpus/validation/indexing.py b/backend/addcorpus/validation/indexing.py index 92d0c89c2..a3763ef7b 100644 --- a/backend/addcorpus/validation/indexing.py +++ b/backend/addcorpus/validation/indexing.py @@ -1,9 +1,9 @@ ''' This module defines functions to check if a corpus is ready for indexing. ''' -import os -import requests + import warnings +import os from addcorpus.validation.creation import primary_mapping_type @@ -87,19 +87,12 @@ def validate_has_data_directory(corpus): return config = corpus.configuration - if not config.data_directory and not config.data_url: - raise CorpusNotIndexableError('Missing data directory or url') - - if corpus.data_dircetory and not os.path.isdir(config.data_directory): - raise CorpusNotIndexableError('Configured data directory does not exist.') + if not config.data_directory: + raise CorpusNotIndexableError( + 'Missing data directory' + ) - if corpus.data_url: - headers = {} - if corpus.data_api_key: - headers = {"Authorization": f"Token {corpus.data_api_key}"} - try: - requests.get(corpus.data_url, headers=headers) - except ConnectionError: - raise CorpusNotIndexableError( - 'Cannot connect to the configured data url. Do you need to provide an API key?' - ) + if not os.path.isdir(config.data_directory): + raise CorpusNotIndexableError( + 'Configured data directory does not exist.' + ) diff --git a/backend/corpora/gallica/conftest.py b/backend/corpora/gallica/conftest.py deleted file mode 100644 index e1d7e2fef..000000000 --- a/backend/corpora/gallica/conftest.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -import pytest - -here = os.path.abspath(os.path.dirname(__file__)) - - -@pytest.fixture() -def gallica_corpus_settings(settings): - settings.CORPORA = { - "figaro": os.path.join(here, "figaro.py"), - } - - -class MockResponse(object): - def __init__(self, filepath): - self.mock_content_file = filepath - - @property - def content(self): - with open(self.mock_content_file, "r") as f: - return f.read() - - -def mock_response(url: str) -> MockResponse: - if url.endswith("date"): - filename = os.path.join(here, "tests", "data", "figaro", "Years.xml") - elif "&" in url: - filename = os.path.join(here, "tests", "data", "figaro", "Issues.xml") - elif "?" in url: - filename = os.path.join(here, "tests", "data", "figaro", "OAIRecord.xml") - elif url.endswith("texteBrut"): - filename = os.path.join(here, "tests", "data", "figaro", "RoughText.html") - return MockResponse(filename) - diff --git a/backend/corpora/gallica/figaro.py b/backend/corpora/gallica/figaro.py deleted file mode 100644 index 32ac4347b..000000000 --- a/backend/corpora/gallica/figaro.py +++ /dev/null @@ -1,55 +0,0 @@ -from datetime import datetime -from typing import Union - -from django.conf import settings -from ianalyzer_readers.xml_tag import Tag -from ianalyzer_readers.extract import XML - -from addcorpus.python_corpora.corpus import FieldDefinition -from addcorpus.es_mappings import ( - keyword_mapping, -) - -from corpora.gallica.gallica import Gallica - - -def join_issue_strings(issue_description: Union[list[str], None]) -> Union[str, None]: - if issue_description: - return "".join(issue_description[:2]) - - -class Figaro(Gallica): - title = "Le Figaro" - description = "Newspaper archive, 1854-1953" - min_date = datetime(year=1854, month=1, day=1) - max_date = datetime(year=1953, month=12, day=31) - corpus_id = "cb34355551z" - category = "periodical" - es_index = getattr(settings, 'FIGARO_INDEX', 'figaro') - image = "figaro.jpg" - - contributor = FieldDefinition( - name="contributor", - description="Persons who contributed to this issue", - es_mapping=keyword_mapping(enable_full_text_search=True), - extractor=XML(Tag("dc:contributor"), multiple=True), - ) - - issue = FieldDefinition( - name="issue", - description="Issue description", - es_mapping=keyword_mapping(), - extractor=XML( - Tag("dc:description"), multiple=True, transform=join_issue_strings - ), - ) - - def __init__(self): - self.fields = [ - self.content(), - self.contributor, - self.date(self.min_date, self.max_date), - self.identifier(), - self.issue, - self.url(), - ] diff --git a/backend/corpora/gallica/gallica.py b/backend/corpora/gallica/gallica.py deleted file mode 100644 index fa52cce58..000000000 --- a/backend/corpora/gallica/gallica.py +++ /dev/null @@ -1,162 +0,0 @@ -from datetime import datetime -import logging -from time import sleep - -from bs4 import BeautifulSoup -from ianalyzer_readers.xml_tag import Tag -from ianalyzer_readers.extract import Metadata, XML -import requests - -from addcorpus.python_corpora.corpus import XMLCorpusDefinition -from addcorpus.python_corpora.corpus import FieldDefinition -from addcorpus.python_corpora.filters import DateFilter -from addcorpus.es_mappings import ( - keyword_mapping, - date_mapping, - main_content_mapping, -) -from addcorpus.es_settings import es_settings - -logger = logging.getLogger('indexing') - -def get_content(content: BeautifulSoup) -> str: - """Return text content in the parsed HTML file from the `texteBrut` request - This is contained in the first

element after the first

element. - """ - text_nodes = content.find("hr").find_next_siblings("p") - return "".join([node.get_text() for node in text_nodes]) - - -def get_publication_id(identifier: str) -> str: - try: - return identifier.split("/")[-1] - except: - return None - - -class Gallica(XMLCorpusDefinition): - - languages = ["fr"] - data_url = "https://gallica.bnf.fr" - corpus_id = "" # each corpus on Gallica has an "ark" id - - @property - def es_settings(self): - return es_settings( - self.languages[:1], stopword_analysis=True, stemming_analysis=True - ) - - def sources(self, start: datetime, end: datetime): - # obtain list of ark numbers - response = requests.get( - f"{self.data_url}/services/Issues?ark=ark:/12148/{self.corpus_id}/date" - ) - year_soup = BeautifulSoup(response.content, "xml") - years = [ - year.string - for year in year_soup.find_all("year") - if int(year.string) >= start.year and int(year.string) <= end.year - ] - for year in years: - try: - response = requests.get( - f"{self.data_url}/services/Issues?ark=ark:/12148/{self.corpus_id}/date&date={year}" - ) - ark_soup = BeautifulSoup(response.content, "xml") - ark_numbers = [ - issue_tag["ark"] for issue_tag in ark_soup.find_all("issue") - ] - sleep(2) - except ConnectionError: - logger.warning(f"Connection error when processing year {year}") - break - - for ark in ark_numbers: - try: - source_response = requests.get( - f"{self.data_url}/services/OAIRecord?ark={ark}" - ) - sleep(2) - except ConnectionError: - logger.warning(f"Connection error encountered in issue {ark}") - break - - if source_response: - try: - content_response = requests.get( - f"{self.data_url}/ark:/12148/{ark}.texteBrut" - ) - sleep(10) - except ConnectionError: - logger.warning( - f"Connection error when fetching full text of issue {ark}" - ) - parsed_content = BeautifulSoup( - content_response.content, "lxml-html" - ) - yield ( - source_response.content, - {"content": parsed_content}, - ) - - def content(self): - return FieldDefinition( - name="content", - description="Content of publication", - display_name="Content", - display_type="text_content", - es_mapping=main_content_mapping( - token_counts=True, - stopword_analysis=True, - stemming_analysis=True, - language=self.languages[0], - ), - extractor=Metadata("content", transform=get_content), - ) - - def date(self, min_date: datetime, max_date: datetime): - return FieldDefinition( - name="date", - display_name="Date", - description="The date of the publication.", - es_mapping=date_mapping(), - extractor=XML( - Tag("dc:date"), - ), - results_overview=True, - search_filter=DateFilter( - min_date, max_date, description="Search only within this time range." - ), - visualizations=["resultscount", "termfrequency"], - csv_core=True, - ) - - def identifier(self): - return FieldDefinition( - name="id", - display_name="Publication ID", - description="Identifier of the publication on Gallica", - es_mapping=keyword_mapping(), - extractor=XML(Tag("dc:identifier"), transform=get_publication_id), - csv_core=True, - ) - - def url(self): - return FieldDefinition( - name="url", - display_name="Source URL", - display_type="url", - description="URL to scan on Gallica", - es_mapping=keyword_mapping(), - extractor=XML(Tag("dc:identifier")), - searchable=False, - ) - - # define fields property so it can be set in __init__ - @property - def fields(self): - return self._fields - - @fields.setter - def fields(self, value): - self._fields = value diff --git a/backend/corpora/gallica/images/figaro.jpg b/backend/corpora/gallica/images/figaro.jpg deleted file mode 100644 index a59554cae..000000000 Binary files a/backend/corpora/gallica/images/figaro.jpg and /dev/null differ diff --git a/backend/corpora/gallica/tests/__init__.py b/backend/corpora/gallica/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/backend/corpora/gallica/tests/data/figaro/Issues.xml b/backend/corpora/gallica/tests/data/figaro/Issues.xml deleted file mode 100644 index 667c8483d..000000000 --- a/backend/corpora/gallica/tests/data/figaro/Issues.xml +++ /dev/null @@ -1,4 +0,0 @@ - - -01 janvier 1930 - diff --git a/backend/corpora/gallica/tests/data/figaro/OAIRecord.xml b/backend/corpora/gallica/tests/data/figaro/OAIRecord.xml deleted file mode 100644 index 2ac45c5a1..000000000 --- a/backend/corpora/gallica/tests/data/figaro/OAIRecord.xml +++ /dev/null @@ -1,62 +0,0 @@ - - -all - - -
-oai:bnf.fr:gallica/ark:/12148/bpt6k296099q -2024-06-21 -gallica:corpus:BnPlCo00 -gallica:corpus:Pam1 -gallica:corpus:bresil -gallica:theme:0:07 -gallica:typedoc:periodiques:fascicules -
- - -https://gallica.bnf.fr/ark:/12148/bpt6k296099q -1930-01-01 -01 janvier 1930 -1930/01/01 (Numéro 1). - -Figaro : journal non politique -Villemessant, Hippolyte de (1810-1879). Directeur de publication -Jouvin, Benoît (1810-1886). Directeur de publication -Figaro (Paris) -texte -text -publication en série imprimée -printed serial -fre -Notice du catalogue : http://catalogue.bnf.fr/ark:/12148/cb34355551z -Bibliothèque nationale de France -domaine public -public domain -http://gallica.bnf.fr/ark:/12148/cb34355551z/date -Appartient à l’ensemble documentaire : BIPFPIG00 -Appartient à l’ensemble documentaire : BIPFPIG63 -Appartient à l’ensemble documentaire : BIPFPIG69 -Appartient à l’ensemble documentaire : Pam1 -Appartient à l’ensemble documentaire : BnPlCo00 -Appartient à l’ensemble documentaire : BnPlCo01 -Appartient à l’ensemble documentaire : FranceBr -Nombre total de vues : 164718 - - - - -bnf.fr -07 -0 -Bibliothèque nationale de France -fascicule -0.0 -Figaro : journal non politique -1930-01-01 -15/10/2007 -false - - - - - diff --git a/backend/corpora/gallica/tests/data/figaro/RoughText.html b/backend/corpora/gallica/tests/data/figaro/RoughText.html deleted file mode 100644 index 4501d9320..000000000 --- a/backend/corpora/gallica/tests/data/figaro/RoughText.html +++ /dev/null @@ -1,2 +0,0 @@ -Figaro : journal non politique | Gallica

Reminder of your request:


Downloading format: : Text

View 1 to 8 on 8

Number of pages: 8

Full notice

Title : Figaro : journal non politique

Publisher : Figaro (Paris)

Publication date : 1930-01-01

Contributor : Villemessant, Hippolyte de (1810-1879). Directeur de publication

Contributor : Jouvin, Benoît (1810-1886). Directeur de publication

Relationship : http://catalogue.bnf.fr/ark:/12148/cb34355551z

Relationship : https://gallica.bnf.fr/ark:/12148/cb34355551z/date

Type : text

Type : printed serial

Language : french

Format : Nombre total de vues : 164718

Description : 01 janvier 1930

Description : 1930/01/01 (Numéro 1).

Description : Collection numérique : Bibliographie de la presse française politique et d'information générale

Description : Collection numérique : BIPFPIG63

Description : Collection numérique : BIPFPIG69

Description : Collection numérique : Arts de la marionnette

Description : Collection numérique : Commun Patrimoine: bibliothèque numérique du réseau des médiathèques de Plaine commune

Description : Collection numérique : La Commune de Paris

Description : Collection numérique : France-Brésil

Rights : Consultable en ligne

Rights : Public domain

Identifier : ark:/12148/bpt6k296099q

Source : Bibliothèque nationale de France

Provenance : Bibliothèque nationale de France

Online date : 15/10/2007

The text displayed may contain some errors. The text of this document has been generated automatically by an optical character recognition (OCR) program. The -estimated recognition rate for this document is 0%.

SOMMAIRE DE FIGARO PAGE 2. Les Cours, les Ambassades, le Monde et la Ville. Les Echos. La fin du Bulletin vert. 1929-1930.

PAGE 3. La Dernière Heure. Avant la Conférence de La Haye. Les méfaits de la tempête.

PAGE 4. La Vie sportive. Revue de la Presse. Anne Douglas Sedgwick Marthe Ludérac.

PAGE 5. Henri Rebois L'Art espagnol à l'Exposition de Barcelone. Robert Brussel Le Mouvement musical. Guy de Passillé Les Etrennes. Jacques Patin Les Premières. Les Alguazils Courrier des Lettres. Marc Hélys Revues étrangères. PAGE 6. La Bourse La Cote des Valeurs. Le Programme des spectacles.

PAGE 7. Courrier des théâtres. Les Courses LA POLITIQUE

La diplomatie

diff --git a/backend/corpora/gallica/tests/data/figaro/Years.xml b/backend/corpora/gallica/tests/data/figaro/Years.xml deleted file mode 100644 index 4ca05725c..000000000 --- a/backend/corpora/gallica/tests/data/figaro/Years.xml +++ /dev/null @@ -1,102 +0,0 @@ - - -1854 -1855 -1856 -1857 -1858 -1859 -1860 -1861 -1862 -1863 -1864 -1865 -1866 -1867 -1868 -1869 -1870 -1871 -1872 -1873 -1874 -1875 -1876 -1877 -1878 -1879 -1880 -1881 -1882 -1883 -1884 -1885 -1886 -1887 -1888 -1889 -1890 -1891 -1892 -1893 -1894 -1895 -1896 -1897 -1898 -1899 -1900 -1901 -1902 -1903 -1904 -1905 -1906 -1907 -1908 -1909 -1910 -1911 -1912 -1913 -1914 -1915 -1916 -1917 -1918 -1919 -1920 -1921 -1922 -1923 -1924 -1925 -1926 -1927 -1928 -1929 -1930 -1931 -1932 -1933 -1934 -1935 -1936 -1937 -1938 -1939 -1940 -1941 -1942 -1944 -1945 -1946 -1947 -1948 -1949 -1950 -1951 -1952 -1953 - diff --git a/backend/corpora/gallica/tests/test_import.py b/backend/corpora/gallica/tests/test_import.py deleted file mode 100644 index 72ad4bb36..000000000 --- a/backend/corpora/gallica/tests/test_import.py +++ /dev/null @@ -1,33 +0,0 @@ -from datetime import datetime -import requests - -from conftest import mock_response -from addcorpus.python_corpora.load_corpus import load_corpus_definition - - -target_documents = [ - { - "content": "SOMMAIRE DE FIGARO PAGE 2. Les Cours, les Ambassades, le Monde et la Ville. Les Echos. La fin du Bulletin vert. 1929-1930. PAGE 3. La Dernière Heure. Avant la Conférence de La Haye. Les méfaits de la tempête. PAGE 4. La Vie sportive. Revue de la Presse. Anne Douglas Sedgwick Marthe Ludérac. PAGE 5. Henri Rebois L'Art espagnol à l'Exposition de Barcelone. Robert Brussel Le Mouvement musical. Guy de Passillé Les Etrennes. Jacques Patin Les Premières. Les Alguazils Courrier des Lettres. Marc Hélys Revues étrangères. PAGE 6. La Bourse La Cote des Valeurs. Le Programme des spectacles. PAGE 7. Courrier des théâtres. Les Courses LA POLITIQUE La diplomatie ", - "contributor": [ - "Villemessant, Hippolyte de (1810-1879). Directeur de publication", - "Jouvin, Benoît (1810-1886). Directeur de publication", - ], - "date": "1930-01-01", - "id": "bpt6k296099q", - "issue": "01 janvier 19301930/01/01 (Numéro 1).", - "url": "https://gallica.bnf.fr/ark:/12148/bpt6k296099q", - } -] - -def test_gallica_import(monkeypatch, gallica_corpus_settings): - monkeypatch.setattr(requests, "get", mock_response) - corpus_def = load_corpus_definition("figaro") - sources = corpus_def.sources( - start=datetime(year=1930, month=1, day=1), - end=datetime(year=1930, month=12, day=31), - ) - documents = list(corpus_def.documents(sources)) - assert len(documents) == 1 - for document, target in zip(documents, target_documents): - for target_key in target.keys(): - assert document.get(target_key) == target.get(target_key) diff --git a/backend/requirements.txt b/backend/requirements.txt index 818de4662..6452878f8 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile @@ -132,7 +132,7 @@ h11==0.14.0 # wsproto humanize==4.9.0 # via flower -ianalyzer-readers==0.2.2 +ianalyzer-readers==0.2.1 # via -r requirements.in idna==3.4 # via @@ -249,9 +249,7 @@ pygments==2.16.1 # rich # seleniumbase pyjwt[crypto]==2.8.0 - # via - # django-allauth - # pyjwt + # via django-allauth pynose==1.4.8 # via seleniumbase pyopenssl==23.2.0 @@ -403,7 +401,10 @@ trio-websocket==0.10.3 # selenium # seleniumbase typing-extensions==4.7.1 - # via asgiref + # via + # asgiref + # kombu + # pypdf2 tzdata==2023.3 # via # celery