✨ catalog: adds services compatibility policy 🗃️ (#6071)

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -237,7 +237,7 @@ class ServiceGetV2(BaseModel):
 
     access_rights: dict[GroupID, ServiceGroupAccessRightsV2] | None
 
-    classifiers: list[str] | None = None
+    classifiers: list[str] | None = []
     quality: dict[str, Any] = {}
 
     history: list[ServiceRelease] = Field(

packages/models-library/src/models_library/services_history.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from typing import Any, ClassVar, TypeAlias
 
+from models_library.services_base import ServiceKeyVersion
 from pydantic import BaseModel, Field
 
 from .services_types import ServiceVersion
@@ -9,7 +10,7 @@
 
 class Compatibility(BaseModel):
     # NOTE: as an object it is more maintainable than a list
-    can_update_to: ServiceVersion = Field(
+    can_update_to: ServiceVersion | ServiceKeyVersion = Field(
         ...,
         description="Latest compatible version at this moment."
         "Current service can update to this version and still work",

packages/postgres-database/src/simcore_postgres_database/migration/versions/d0e56c2d0a0d_new_services_comp_table.py

@@ -0,0 +1,97 @@
+"""new services_comp table
+
+Revision ID: d0e56c2d0a0d
+Revises: 19f3d9085636
+Create Date: 2024-07-17 16:15:49.970615+00:00
+
+"""
+from typing import Final
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "d0e56c2d0a0d"
+down_revision = "19f3d9085636"
+branch_labels = None
+depends_on = None
+
+# auto-update modified
+# TRIGGERS ------------------------
+_TABLE_NAME: Final[str] = "services_compatibility"
+_TRIGGER_NAME: Final[str] = "trigger_auto_update"  # NOTE: scoped on table
+_PROCEDURE_NAME: Final[
+    str
+] = f"{_TABLE_NAME}_auto_update_modified()"  # NOTE: scoped on database
+modified_timestamp_trigger = sa.DDL(
+    f"""
+DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};
+CREATE TRIGGER {_TRIGGER_NAME}
+BEFORE INSERT OR UPDATE ON {_TABLE_NAME}
+FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME};
+    """
+)
+
+# PROCEDURES ------------------------
+update_modified_timestamp_procedure = sa.DDL(
+    f"""
+CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME}
+RETURNS TRIGGER AS $$
+BEGIN
+  NEW.modified := current_timestamp;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+    """
+)
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "services_compatibility",
+        sa.Column("key", sa.String(), nullable=False),
+        sa.Column("version", sa.String(), nullable=False),
+        sa.Column(
+            "custom_policy", postgresql.JSONB(astext_type=sa.Text()), nullable=False
+        ),
+        sa.Column(
+            "created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "modified",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column("modified_by", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["key", "version"],
+            ["services_meta_data.key", "services_meta_data.version"],
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["modified_by"], ["users.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("key", "version", name="services_compatibility_pk"),
+    )
+    # ### end Alembic commands ###
+
+    # custom
+    op.execute(update_modified_timestamp_procedure)
+    op.execute(modified_timestamp_trigger)
+
+
+def downgrade():
+    # custom
+    op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};")
+    op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};")
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("services_compatibility")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/_common.py

@@ -26,6 +26,38 @@ def column_modified_datetime(*, timezone: bool = True) -> sa.Column:
     )
 
 
+def column_created_by_user(
+    *, users_table: sa.Table, required: bool = False
+) -> sa.Column:
+    return sa.Column(
+        "created_by",
+        sa.Integer,
+        sa.ForeignKey(
+            users_table.c.id,
+            onupdate="CASCADE",
+            ondelete="SET NULL",
+        ),
+        nullable=not required,
+        doc="Who created this row at `created`",
+    )
+
+
+def column_modified_by_user(
+    *, users_table: sa.Table, required: bool = False
+) -> sa.Column:
+    return sa.Column(
+        "modified_by",
+        sa.Integer,
+        sa.ForeignKey(
+            users_table.c.id,
+            onupdate="CASCADE",
+            ondelete="SET NULL",
+        ),
+        nullable=not required,
+        doc="Who modified this row at `modified`",
+    )
+
+
 _TRIGGER_NAME: Final[str] = "auto_update_modified_timestamp"
 
 

packages/postgres-database/src/simcore_postgres_database/models/services.py

@@ -4,20 +4,21 @@
     - Services have a key, version, and access rights defined by group ids
 """
 
+
 import sqlalchemy as sa
 from sqlalchemy import null
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.sql import expression, func
 
 from .base import metadata
 
-#
-#   Combines properties as
-#     - service identifier: key, version
-#     - overridable properties of the service metadata defined upon publication (injected in the image labels)
-#     - extra properties assigned during its lifetime (e.g. deprecated, quality, etc)
-
 services_meta_data = sa.Table(
+    #
+    #   Combines properties as
+    #     - service identifier: key, version
+    #     - overridable properties of the service metadata defined upon publication (injected in the image labels)
+    #     - extra properties assigned during its lifetime (e.g. deprecated, quality, etc)
+    #
     "services_meta_data",
     metadata,
     sa.Column(
@@ -103,12 +104,11 @@
 )
 
 
-#
-#   Defines access rights (execute_access, write_access) on a service (key)
-#   for a given group (gid) on a product (project_name)
-#
-
 services_access_rights = sa.Table(
+    #
+    #   Defines access rights (execute_access, write_access) on a service (key)
+    #   for a given group (gid) on a product (project_name)
+    #
     "services_access_rights",
     metadata,
     sa.Column(

packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py

@@ -0,0 +1,68 @@
+""" Services table
+
+    - List of 3rd party services in the framework
+    - Services have a key, version, and access rights defined by group ids
+"""
+
+
+import sqlalchemy as sa
+import typing_extensions
+from sqlalchemy.dialects.postgresql import JSONB
+from typing_extensions import NotRequired, Required
+
+from ._common import (
+    column_created_datetime,
+    column_modified_by_user,
+    column_modified_datetime,
+)
+from .base import metadata
+from .users import users
+
+
+class CompatiblePolicyDict(typing_extensions.TypedDict, total=False):
+    # SpecifierSet e.g. ~=0.9
+    # SEE https://packaging.python.org/en/latest/specifications/version-specifiers/#id5
+    versions_specifier: Required[str]
+    # Only necessary if key!=PolicySpecifierDict.key
+    other_service_key: NotRequired[str | None]
+
+
+services_compatibility = sa.Table(
+    #
+    # CUSTOM COMPATIBILITY POLICIES
+    # Otherwise default compatibility policy is employed.
+    #
+    "services_compatibility",
+    metadata,
+    sa.Column(
+        "key",
+        sa.String,
+        nullable=False,
+        doc="Service Key Identifier",
+    ),
+    sa.Column(
+        "version",
+        sa.String,
+        nullable=False,
+        doc="Service version",
+    ),
+    sa.Column(
+        "custom_policy",
+        JSONB,
+        nullable=False,
+        doc="PolicySpecifierDict with custom policy",
+    ),
+    # Traceability, i.e. when
+    column_created_datetime(timezone=True),
+    column_modified_datetime(timezone=True),
+    # Traceability, i.e. who
+    column_modified_by_user(users_table=users, required=True),
+    # Constraints
+    sa.ForeignKeyConstraint(
+        ["key", "version"],
+        ["services_meta_data.key", "services_meta_data.version"],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+    ),
+    sa.PrimaryKeyConstraint("key", "version", name="services_compatibility_pk"),
+)

services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py

@@ -7,7 +7,13 @@
 from sqlalchemy.sql.expression import func
 from sqlalchemy.sql.selectable import Select
 
-from ..tables import services_access_rights, services_meta_data, user_to_groups, users
+from ..tables import (
+    services_access_rights,
+    services_compatibility,
+    services_meta_data,
+    user_to_groups,
+    users,
+)
 
 
 def list_services_stmt(
@@ -172,6 +178,7 @@ def list_latest_services_with_history_stmt(
             services_meta_data.c.version,
             services_meta_data.c.deprecated,
             services_meta_data.c.created,
+            services_compatibility.c.custom_policy,  # CompatiblePolicyDict | None
         )
         .select_from(
             services_meta_data.join(
@@ -184,11 +191,17 @@ def list_latest_services_with_history_stmt(
                 (services_meta_data.c.key == services_access_rights.c.key)
                 & (services_meta_data.c.version == services_access_rights.c.version)
                 & (services_access_rights.c.product_name == product_name),
-            ).join(
+            )
+            .join(
                 user_to_groups,
                 (user_to_groups.c.gid == services_access_rights.c.gid)
                 & (user_to_groups.c.uid == user_id),
             )
+            .outerjoin(
+                services_compatibility,
+                (services_meta_data.c.key == services_compatibility.c.key)
+                & (services_meta_data.c.version == services_compatibility.c.version),
+            )
         )
         .where(access_rights)
         .order_by(
@@ -224,6 +237,8 @@ def list_latest_services_with_history_stmt(
                     history_subquery.c.deprecated,
                     "created",
                     history_subquery.c.created,
+                    "compatibility_policy",  # NOTE: this is the `policy`
+                    history_subquery.c.custom_policy,
                 )
             ).label("history"),
         )
@@ -316,17 +331,24 @@ def get_service_history_stmt(
             services_meta_data.c.version,
             services_meta_data.c.deprecated,
             services_meta_data.c.created,
+            services_compatibility.c.custom_policy,  # CompatiblePolicyDict | None
         )
         .select_from(
             # joins because access-rights might change per version
             services_meta_data.join(
                 services_access_rights,
                 (services_meta_data.c.key == services_access_rights.c.key)
                 & (services_meta_data.c.version == services_access_rights.c.version),
-            ).join(
+            )
+            .join(
                 user_to_groups,
                 (user_to_groups.c.gid == services_access_rights.c.gid),
             )
+            .outerjoin(
+                services_compatibility,
+                (services_meta_data.c.key == services_compatibility.c.key)
+                & (services_meta_data.c.version == services_compatibility.c.version),
+            )
         )
         .where(
             (services_meta_data.c.key == service_key)
@@ -338,18 +360,24 @@ def get_service_history_stmt(
             services_meta_data.c.key,
             sa.desc(_version(services_meta_data.c.version)),  # latest version first
         )
-        .subquery()
+        .alias("history_subquery")
     )
 
-    return sa.select(
-        array_agg(
-            func.json_build_object(
-                "version",
-                history_subquery.c.version,
-                "deprecated",
-                history_subquery.c.deprecated,
-                "created",
-                history_subquery.c.created,
-            )
-        ).label("history"),
-    ).group_by(history_subquery.c.key)
+    return (
+        sa.select(
+            array_agg(
+                func.json_build_object(
+                    "version",
+                    history_subquery.c.version,
+                    "deprecated",
+                    history_subquery.c.deprecated,
+                    "created",
+                    history_subquery.c.created,
+                    "compatibility_policy",  # NOTE: this is the `policy`
+                    history_subquery.c.custom_policy,
+                )
+            ).label("history"),
+        )
+        .select_from(history_subquery)
+        .group_by(history_subquery.c.key)
+    )