Merge branch 'master' into lifelike-staging-az-deployment

.github/actions/client-lint-autofix/action.yml

@@ -27,9 +27,11 @@ runs:
         cache-dependency-path: client/yarn.lock
 
     - name: Install dependencies
+      working-directory: client
       run: yarn install --frozen-lockfile
 
     - name: Run linter
+      working-directory: client
       run: yarn lint-ci --fix
 
     - name: Commit and push changes

.github/workflows/annotate-code-style.yml

@@ -83,5 +83,4 @@ jobs:
 
       - name: Client linting autofix
         if: github.event.inputs.autoFix && github.event.inputs.tslintAutoFix
-        working-directory: client
         uses: ./.github/actions/client-lint-autofix

appserver/app.py

@@ -18,7 +18,7 @@
 import timeflake
 from flask import request, g
 from marshmallow.exceptions import ValidationError
-from sqlalchemy import inspect, Table
+from sqlalchemy import inspect, Table, select
 from sqlalchemy.sql.expression import and_, text
 
 from neo4japp.blueprints.auth import auth
@@ -98,18 +98,52 @@ def request_navigator_log():
 login_required_dummy_view = auth.login_required(lambda: None)
 
 
+def login_optional_dummy_view():
+    try:
+        _auth = auth.get_auth()
+
+        # Flask normally handles OPTIONS requests on its own, but in the
+        # case it is configured to forward those to the application, we
+        # need to ignore authentication headers and let the request through
+        # to avoid unwanted interactions with CORS.
+        if request.method != 'OPTIONS' and _auth:  # pragma: no cover
+            password = auth.get_auth_password(_auth)
+
+            auth.authenticate(_auth, password)
+    except Exception:
+        # We intentionally want to swallow errors here to allow the
+        # application to handle unauthorized access in a custom manner
+        # e.g. through a @login_optional decorated view.
+        pass
+
+
+def route_has_flag(flag: str):
+    view = app.view_functions[request.endpoint]
+
+    if getattr(view, flag, False):
+        return True
+    view_class = getattr(view, 'view_class', None)
+    if view_class:
+        if getattr(view_class, flag, False):
+            return True
+        method = getattr(view_class, request.method.lower(), None)
+        if method and getattr(method, flag, False):
+            return True
+
+
 @app.before_request
 def default_login_required():
     # exclude 404 errors and static routes
     # uses split to handle blueprint static routes as well
     if not request.endpoint or request.endpoint.rsplit('.', 1)[-1] == 'static':
         return
 
-    view = app.view_functions[request.endpoint]
-
-    if getattr(view, 'login_exempt', False):
+    if route_has_flag('login_exempt'):
         return
 
+    if route_has_flag('login_optional'):
+        return login_optional_dummy_view()
+
     return login_required_dummy_view()
 
 
@@ -442,6 +476,74 @@ def upload_lmdb():
     manager.upload_all(lmdb_dir_path)
 
 
+@app.cli.command('test-prepublish')
+@click.option('--email', '-e', required=True, type=str)
+@click.option('--password', '-p', required=True, type=str)
+def test_prepublish(email, password):
+    from tests.helpers.api import generate_jwt_headers
+
+    client = app.test_client()
+    login_resp = client.post(
+        '/auth/login',
+        data=json.dumps({'email': email, 'password': password}),
+        content_type='application/json',
+    ).get_json()
+
+    headers = generate_jwt_headers(login_resp['accessToken']['token'])
+
+    user = db.session.query(AppUser).filter(AppUser.email == email).one()
+
+    from neo4japp.services.filesystem import Filesystem
+
+    with app.app_context():
+        g.current_user = user
+        for file in Filesystem.get_nondeleted_recycled_files(
+            Files.mime_type == FILE_MIME_TYPE_MAP, lazy_load_content=True
+        ):
+            # print(file, file.calculated_privileges)
+            try:
+                Filesystem.check_file_permissions(
+                    [file], user, ['readable'], permit_recycled=True
+                )
+            except Exception:
+                # Ignore files that are not readable
+                continue
+
+            try:
+                print("Processing file: ", file.path, file.hash_id, file.id)
+                resp = client.post(f'/publish/{file.hash_id}/prepare', headers=headers)
+                if resp.status_code == 200:
+                    print(
+                        "Success", resp.content_length, file.path, file.hash_id, file.id
+                    )
+                    pass
+                else:
+                    print("Failed", file.path, file.hash_id, file.id)
+                    resp_json = resp.get_json()
+                    print(resp.status_code, resp_json)
+                    if resp_json.get('type') == 'FileNotFound':
+                        raise FileNotFoundError
+                    else:
+                        print(
+                            "Failed response",
+                            resp_json,
+                            file.path,
+                            file.hash_id,
+                            file.id,
+                        )
+                        break
+            except FileNotFoundError:
+                print("File not found: ", file.path, file.hash_id, file.id)
+                continue
+            except Exception:
+                print("Exception: ", file.path, file.hash_id, file.id)
+                # printing stack trace
+                import traceback
+
+                traceback.print_exc()
+                raise
+
+
 @app.cli.command('create-lmdb')
 @click.option('--file-type', type=str)
 def create_lmdb_files(file_type):
@@ -677,7 +779,9 @@ def add_file(
         for trial in range(4):
             if 1 <= trial <= 2:  # Try adding (N+1)
                 try:
-                    file.filename = file.generate_non_conflicting_filename()
+                    file.filename = file.generate_non_conflicting_filename(
+                        file.filename
+                    )
                 except ValueError:
                     raise ValidationError(
                         'Filename conflicts with an existing file in the same folder.',

appserver/config.py

@@ -129,7 +129,7 @@ class Base:
 class Development(Base):
     """Development configurations"""
 
-    DOMAIN = 'http://localhost'
+    DOMAIN = 'http://localhost:4200'
 
     LOGGING_LEVEL = logging.DEBUG
 

appserver/migrations/versions/2023_11_02-0f510c3ffec4_.py

@@ -47,6 +47,7 @@
 
 BATCH_SIZE = 1
 
+
 def fix_regulon_links(enrichment: dict) -> dict:
     result = enrichment.get('result', None)
     if result is not None:
@@ -62,11 +63,12 @@ def fix_regulon_links(enrichment: dict) -> dict:
                             if link is not None:
                                 if link.startswith('http://'):
                                     new_link = f'https://{link.split("http://")[1]}'
-                                    enrichment['result']['genes'][idx]['domains']['Regulon'][subdomain]['link'] = new_link
+                                    enrichment['result']['genes'][idx]['domains'][
+                                        'Regulon'
+                                    ][subdomain]['link'] = new_link
     return enrichment
 
 
-
 def upgrade():
     if context.get_x_argument(as_dictionary=True).get('data_migrate', None):
         data_upgrades()
@@ -96,8 +98,8 @@ def data_upgrades():
                 t_files,
                 sa.and_(
                     t_files.c.content_id == t_files_content.c.id,
-                    t_files.c.mime_type == 'vnd.***ARANGO_DB_NAME***.document/enrichment-table'
-                )
+                    t_files.c.mime_type == 'vnd.***ARANGO_DB_NAME***.document/enrichment-table',
+                ),
             )
         )
         .order_by(t_files.c.id)
@@ -108,25 +110,28 @@ def data_upgrades():
             if annotations_obj not in [[], '[]']:
                 print(f'Processing File#{file_id}')
                 enrichment_annotations = json.load(BytesIO(raw_file))
-                updated_enrichment_annotations = fix_regulon_links(enrichment_annotations)
+                updated_enrichment_annotations = fix_regulon_links(
+                    enrichment_annotations
+                )
                 updated_raw_file = BytesIO()
-                updated_raw_file.write(json.dumps(updated_enrichment_annotations).encode())
+                updated_raw_file.write(
+                    json.dumps(updated_enrichment_annotations).encode()
+                )
                 updated_raw_file = updated_raw_file.getvalue()
                 new_hash = hashlib.sha256(updated_raw_file).digest()
 
                 existing_file_content = session.execute(
-                    sa.select(
-                        [t_files_content.c.id]
-                    )
-                    .where(
+                    sa.select([t_files_content.c.id]).where(
                         t_files_content.c.checksum_sha256 == new_hash,
                     )
                 ).first()
 
                 # If the checksum already exists in the FileContent table, don't update the old
                 # row. Instead, update the File row with the existing FileContent id.
                 if existing_file_content is not None:
-                    print(f'\tUsing new content id for Files#{file_id}: {existing_file_content[0]}')
+                    print(
+                        f'\tUsing new content id for Files#{file_id}: {existing_file_content[0]}'
+                    )
                     content_id = existing_file_content[0]
                 else:
                     print(f'\tUpdating FilesContent#{content_id}')

appserver/neo4japp/blueprints/annotations.py

@@ -28,7 +28,7 @@
 
 from neo4japp.exceptions import AnnotationError, ServerException
 from neo4japp.exceptions import wrap_exceptions
-from .auth import login_exempt
+from .auth import login_exempt, login_optional
 from .filesystem import bp as filesystem_bp, FilesystemBaseView
 from .permissions import requires_role
 from .utils import get_missing_hash_ids
@@ -93,6 +93,7 @@
 from ..services.arangodb import convert_datetime, execute_arango_query, get_db
 from ..services.enrichment.data_transfer_objects import EnrichmentCellTextMapping
 from ..services.filesystem import Filesystem
+from ..utils.globals import get_current_user
 from ..utils.http import make_cacheable_file_response
 from ..utils.logger import UserEventLog
 from ..utils.string import sub_whitespace
@@ -101,15 +102,14 @@
 
 
 class FileAnnotationsView(FilesystemBaseView):
+    @login_optional
     def get(self, hash_id: str):
         """Fetch annotations for a file.."""
-        current_user = g.current_user
-
         file = Filesystem.get_nondeleted_recycled_file(
             Files.hash_id == hash_id, lazy_load_content=True
         )
         Filesystem.check_file_permissions(
-            [file], current_user, ['readable'], permit_recycled=True
+            [file], get_current_user(), ['readable'], permit_recycled=True
         )
 
         if file.annotations:
@@ -149,15 +149,14 @@ def terms_match(term_in_exclusion, term_in_annotation, is_case_insensitive):
 
 
 class EnrichmentAnnotationsView(FilesystemBaseView):
+    @login_optional
     def get(self, hash_id: str):
         """Fetch annotations for enrichment table."""
-        current_user = g.current_user
-
         file = Filesystem.get_nondeleted_recycled_file(
             Files.hash_id == hash_id, lazy_load_content=True
         )
         Filesystem.check_file_permissions(
-            [file], current_user, ['readable'], permit_recycled=True
+            [file], get_current_user(), ['readable'], permit_recycled=True
         )
 
         if file.enrichment_annotations:
@@ -349,6 +348,7 @@ def get_rows(self, files, annotation_service):
 
         yield from get_sorted_annotation_rows(values, 'value')
 
+    @login_optional
     @use_args(
         {
             "sort": fields.Str(
@@ -360,13 +360,12 @@ def get_rows(self, files, annotation_service):
     )
     def post(self, args: Dict[str, str], hash_id: str):
         sort = args['sort']
-        current_user = g.current_user
 
         file = Filesystem.get_nondeleted_recycled_file(
             Files.hash_id == hash_id, lazy_load_content=True
         )
         Filesystem.check_file_permissions(
-            [file], current_user, ['readable'], permit_recycled=True
+            [file], get_current_user(), ['readable'], permit_recycled=True
         )
 
         buffer = io.StringIO()

appserver/neo4japp/blueprints/auth.py

@@ -44,6 +44,14 @@ def login_exempt(f):
     return f
 
 
+def login_optional(f):
+    """
+    Decorator used to specify endpoints that do not require user authentication but use it.
+    """
+    f.login_optional = True
+    return f
+
+
 class TokenService:
     def __init__(self, app_secret: str, algorithm: str = 'HS256'):
         self.app_secret = app_secret

appserver/neo4japp/blueprints/context.py

@@ -5,7 +5,7 @@
 
 from neo4japp.schemas.context import ContextRelationshipRequestSchema
 from neo4japp.services.chat_gpt import ChatGPT
-from neo4japp.utils.globals import current_username
+from neo4japp.utils.globals import get_current_username
 
 bp = Blueprint('chat-gpt-api', __name__, url_prefix='/explain')
 
@@ -20,7 +20,7 @@ def compose_query(entities: List[str], context: Optional[str]):
     else:
         query_core = f'What is the relationship between {", ".join(entities)}'
 
-    context_query = f' in the context of {context}' if context else''
+    context_query = f' in the context of {context}' if context else ''
 
     return query_core + context_query + '?'
 
@@ -41,7 +41,7 @@ def relationship(params):
         ],
         temperature=options.get('temperature', 0),
         max_tokens=2000,
-        user=str(hash(current_username)),
+        user=str(hash(get_current_username())),
         timeout=60,
     )
     response = ChatGPT.ChatCompletion.create(**create_params)

appserver/neo4japp/blueprints/enrichment_visualisation.py

@@ -4,6 +4,7 @@
 import requests
 from flask import Blueprint, Response, current_app, request
 
+from neo4japp.blueprints.auth import login_optional
 from neo4japp.exceptions import StatisticalEnrichmentError, wrap_exceptions
 from neo4japp.services.chat_gpt import ChatGPT
 from neo4japp.utils.globals import config
@@ -81,6 +82,7 @@ def forward_request():
 
 
 @bp.route('/enrich-with-go-terms', methods=['POST'])
+@login_optional
 @wrap_exceptions(StatisticalEnrichmentError)
 def enrich_go():
     return forward_request()