Merge branch 'fix-build'

.github/workflows/ci-build.yml

@@ -29,6 +29,6 @@ jobs:
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-#    - name: Test with pytest
-#      run: |
-#        pytest
+    - name: Test with pytest
+      run: |
+        pytest

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+cache_dir = build/pytest
+testpaths = tests
+markers =
+    slow: marks tests as slow (deselect with '-m "not slow"')

requirements.txt

@@ -6,4 +6,5 @@ gensim
 pandas
 regex
 bump-my-version
-supermat
+supermat
+pytest

scripts/xml2LossyJSON.py

@@ -0,0 +1,257 @@
+import argparse
+import json
+import os
+import re
+from collections import OrderedDict
+from pathlib import Path
+
+from bs4 import BeautifulSoup, NavigableString, Tag
+from supermat.grobid_tokenizer import tokenizeSimple
+from supermat.supermat_tei_parser import get_children_list, get_section, get_hash
+
+
+def tokenise(string):
+    return tokenizeSimple(string)
+
+
+def write_on_file(fw, paragraphText, dic_token, i, item_length):
+    # tsvText += f'#Text={paragraphText}\n'
+    print(f'#Text={paragraphText}', file=fw)
+    for k, v in dic_token.items():
+        # print(v)
+        if k[0] == i + 1 and v[2]:
+            print('{}-{}\t{}-{}\t{}\t{}\t{}\t{}\t{}\t{}\t'.format(*k, *v), file=fw)
+
+    # Avoid adding a line break too much at the end of the file
+    if i != item_length - 1:
+        print('', file=fw)
+
+
+def process_file(finput, use_paragraphs=False):
+    with open(finput, encoding='utf-8') as fp:
+        doc = fp.read()
+
+    mod_tags = re.finditer(r'(</\w+>) ', doc)
+    for mod in mod_tags:
+        doc = doc.replace(mod.group(), ' ' + mod.group(1))
+    soup = BeautifulSoup(doc, 'xml')
+
+    children = get_children_list(soup, use_paragraphs=use_paragraphs)
+
+    off_token = 0
+    ient = 1
+
+    # list containing text and the dictionary with all the annotations
+    paragraphs = []
+    dic_dest_relationships = {}
+    dic_source_relationships = {}
+
+    output_document = OrderedDict()
+    output_document['lang'] = 'en'
+    output_document['level'] = 'sentence' if not use_paragraphs else 'paragraph'
+    output_document['paragraphs'] = paragraphs
+
+    linked_entity_registry = {}
+
+    i = 0
+    for child in children:
+        for pTag in child:
+            paragraph = OrderedDict()
+            j = 0
+            offset = 0
+            section = get_section(pTag)
+            if not section:
+                section = get_section(pTag.parent)
+
+            paragraph['section'] = section
+            paragraph_text = ''
+            paragraph['text'] = paragraph_text
+            spans = []
+            paragraph['spans'] = spans
+            tokens = []
+            paragraph['tokens'] = tokens
+            for item in pTag.contents:
+                if type(item) == NavigableString:
+                    local_text = str(item)
+                    paragraph_text += local_text
+                    offset += len(local_text)
+
+                elif type(item) is Tag and item.name == 'rs':
+                    local_text = item.text
+                    paragraph_text += local_text
+
+                    span = OrderedDict()
+                    front_offset = 0
+                    if local_text.startswith(" "):
+                        front_offset = len(local_text) - len(local_text.lstrip(" "))
+
+                    span['text'] = local_text.strip(" ")
+                    span['offset_start'] = offset + front_offset
+                    span['offset_end'] = offset + len(span['text']) + front_offset
+                    spans.append(span)
+
+                    offset += len(local_text)
+
+                    assert paragraph_text[span['offset_start']:span['offset_end']] == span['text']
+
+                    if 'type' not in item.attrs:
+                        raise Exception("RS without type is invalid. Stopping")
+
+                    entity_class = item.attrs['type']
+                    span['type'] = '<' + entity_class + '>'
+
+                    if len(item.attrs) > 0:
+                        ## multiple entities can point ot the same one, so "corresp" value can be duplicated
+                        allow_duplicates = False
+                        if 'xml:id' in item.attrs:
+                            span['id'] = item['xml:id']
+                            if item.attrs['xml:id'] not in dic_dest_relationships:
+                                dic_dest_relationships[item.attrs['xml:id']] = [i + 1, j + 1, ient, entity_class]
+
+                        if 'corresp' in item.attrs:
+                            if 'id' not in span or span['id'] == "":
+                                id_str = str(i + 1) + "," + str(j + 1)
+                                span['id'] = get_hash(id_str)
+                                if (span['id']) not in dic_source_relationships:
+                                    dic_source_relationships[span['id']] = [item.attrs['corresp'].replace('#', ''),
+                                                                            ient,
+                                                                            entity_class]
+                            else:
+                                if (span['id']) not in dic_source_relationships:
+                                    dic_source_relationships[span['id']] = [item.attrs['corresp'].replace('#', ''),
+                                                                            ient,
+                                                                            entity_class]
+
+                            allow_duplicates = True
+
+                        if 'id' in span:
+                            if span['id'] not in linked_entity_registry.keys():
+                                linked_entity_registry[span['id']] = span
+                            else:
+                                if not allow_duplicates:
+                                    print("The same key exists... something's wrong: ", span['id'])
+
+                    j += 1
+
+                ient += 1  # entity No.
+
+            paragraph['text'] = paragraph_text
+            off_token += 1  # return
+
+            paragraphs.append(paragraph)
+            i += 1
+
+    for id__ in dic_source_relationships:
+        destination_xml_id = dic_source_relationships[id__][0]
+        # source_entity_id = dic_source_relationships[par_num, token_num][1]
+        # label_source = dic_source_relationships[id__][2]
+
+        # destination_xml_id: Use this to pick up information from dic_dest_relationship
+
+        for des in destination_xml_id.split(","):
+            destination_item = dic_dest_relationships[str(des)]
+            # destination_paragraph_tsv = destination_item[0]
+            # destination_token_tsv = destination_item[1]
+            # destination_entity_id = destination_item[2]
+            # label_destination = destination_item[3]
+
+            # relationship_name = get_relationship_name(label, destination_label)
+
+            dict_coordinates = get_hash(id__)
+
+            span_destination = linked_entity_registry[des]
+            span_source = linked_entity_registry[dict_coordinates]
+            link_source = {
+                "targetId": span_destination['id'],
+                "targetText": span_destination['text'],
+                "targetType": span_destination['type']
+            }
+
+            link_destination = {
+                "targetId": span_source['id'],
+                "targetText": span_source['text'],
+                "targetType": span_source['type']
+            }
+
+            if 'links' in span_source:
+                span_source['links'].append(link_source)
+            else:
+                span_source['links'] = [link_source]
+
+            if 'links' in span_destination:
+                span_destination['links'].append(link_destination)
+            else:
+                span_destination['links'] = [link_destination]
+
+    return output_document
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Converter from XML (Grobid training data based on TEI) to lossy JSON (CORD-19) format")
+
+    parser.add_argument("--input",
+                        help="Input file or directory",
+                        required=True)
+    parser.add_argument("--output",
+                        required=True,
+                        help="Output directory")
+    parser.add_argument("--recursive",
+                        action="store_true",
+                        default=False,
+                        help="Process input directory recursively. If input is a file, this parameter is ignored. ")
+    parser.add_argument("--use-paragraphs",
+                        action="store_true",
+                        default=False,
+                        help="Use paragraphs instead of sentences.")
+
+    args = parser.parse_args()
+
+    input = args.input
+    output = args.output
+    recursive = args.recursive
+    use_paragraphs = args.use_paragraphs
+
+    if os.path.isdir(input):
+        path_list = []
+        output_path_list = []
+
+        if recursive:
+            for root, dirs, files in os.walk(input):
+                for dir in dirs:
+                    abs_path_dir = os.path.join(root, dir)
+                    output_path = abs_path_dir.replace(str(input.rstrip("/")), str(output))
+                    os.makedirs(output_path, exist_ok=True)
+
+                for file_ in files:
+                    if not file_.lower().endswith(".tei.xml"):
+                        continue
+
+                    file_input_path = os.path.join(root, file_)
+                    output_path = file_input_path.replace(str(input.rstrip("/")), str(output))
+                    file_output_path = output_path.replace(".xml", ".json").replace(".tei", "")
+                    path_list.append([file_input_path, file_output_path])
+
+        else:
+            input_path_list = list(Path(input).glob('*.tei.xml'))
+            output_path_list = [str(input_path)
+                           .replace(str(input), str(output))
+                           .replace(".xml", ".json")
+                           .replace(".tei", "") for input_path
+                           in input_path_list]
+
+            path_list = list(zip(input_path_list, output_path_list))
+
+        for file_input_path, file_output_path  in path_list:
+            print("Processing: ", file_input_path)
+
+            output_document = process_file(str(file_input_path), use_paragraphs)
+            with open(file_output_path, 'w') as fp:
+                json.dump(output_document, fp)
+
+    elif os.path.isfile(input):
+        input_path = Path(input)
+        output_filename = os.path.join(output, input_path.stem + ".json")
+        output_document = process_file(input_path, use_paragraphs)
+        with open(output_filename, 'w') as fp:
+            json.dump(output_document, fp)