From 1568e3848af92fff7da19b2efcaba5ac013259dc Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Tue, 19 Mar 2024 15:02:30 +0100 Subject: [PATCH] chore: metadata model reorganize pt. 2 --- .../interfaces/python_api/models/info.py | 156 +++++++++++++----- .../models/values/value_metadata/__init__.py | 103 ------------ src/kiara/utils/metadata.py | 54 +++++- 3 files changed, 168 insertions(+), 145 deletions(-) diff --git a/src/kiara/interfaces/python_api/models/info.py b/src/kiara/interfaces/python_api/models/info.py index e53d33750..327efe52b 100644 --- a/src/kiara/interfaces/python_api/models/info.py +++ b/src/kiara/interfaces/python_api/models/info.py @@ -80,7 +80,7 @@ from kiara.operations import OperationType from kiara.registries.aliases import AliasRegistry from kiara.registries.data import DataRegistry - from kiara.models.values.value_metadata import MetadataTypeClassesInfo + from kiara.models.values.value_metadata import ValueMetadata INFO_BASE_INSTANCE_TYPE = TypeVar("INFO_BASE_INSTANCE_TYPE") INFO_BASE_CLASS = TypeVar("INFO_BASE_CLASS", bound=type) @@ -791,6 +791,88 @@ def _retrieve_data_to_hash(self) -> Any: return self.python_class.full_name +class MetadataTypeInfo(TypeInfo): + + _kiara_model_id: ClassVar = "info.metadata_type" + + @classmethod + def create_from_type_class( + self, type_cls: Type["ValueMetadata"], kiara: "Kiara" + ) -> "MetadataTypeInfo": + + authors_md = AuthorsMetadataModel.from_class(type_cls) + doc = DocumentationMetadataModel.from_class_doc(type_cls) + python_class = PythonClass.from_class(type_cls) + properties_md = ContextMetadataModel.from_class(type_cls) + type_name = type_cls._metadata_key # type: ignore + schema = type_cls.model_json_schema() + + return MetadataTypeInfo( + type_name=type_name, + documentation=doc, + authors=authors_md, + context=properties_md, + python_class=python_class, + metadata_schema=schema, + ) + + @classmethod + def base_class(self) -> Type["ValueMetadata"]: + from kiara.models.values.value_metadata import ValueMetadata + return ValueMetadata + + @classmethod + def category_name(cls) -> str: + return "value_metadata" + + metadata_schema: Dict[str, Any] = Field( + description="The (json) schema for this metadata value." + ) + + def create_renderable(self, **config: Any) -> RenderableType: + + include_doc = config.get("include_doc", True) + include_schema = config.get("include_schema", True) + + table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) + table.add_column("property", style="i") + table.add_column("value") + + if include_doc: + table.add_row( + "Documentation", + Panel(self.documentation.create_renderable(), box=box.SIMPLE), + ) + table.add_row("Author(s)", self.authors.create_renderable()) + table.add_row("Context", self.context.create_renderable()) + + if hasattr(self, "python_class"): + table.add_row("Python class", self.python_class.create_renderable()) + + if include_schema: + schema = Syntax( + orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2), + "json", + background_color="default", + ) + table.add_row("metadata_schema", schema) + + return table + + +class MetadataTypeClassesInfo(TypeInfoItemGroup): + + _kiara_model_id: ClassVar = "info.metadata_types" + + @classmethod + def base_info_class(cls) -> Type[TypeInfo]: + return MetadataTypeInfo + + type_name: Literal["value_metadata"] = "value_metadata" + item_infos: Mapping[str, MetadataTypeInfo] = Field( # type: ignore + description="The value metadata info instances for each type." + ) + class DataTypeClassInfo(TypeInfo[Type["DataType"]]): _kiara_model_id: ClassVar = "info.data_type" @@ -800,13 +882,14 @@ def create_from_type_class( self, type_cls: Type["DataType"], kiara: Union["Kiara", None] = None ) -> "DataTypeClassInfo": - from kiara.utils.metadata import find_metadata_models + + from kiara.utils.metadata import get_metadata_model_for_data_type authors = AuthorsMetadataModel.from_class(type_cls) doc = DocumentationMetadataModel.from_class_doc(type_cls) properties_md = ContextMetadataModel.from_class(type_cls) - metadata_models = find_metadata_models() + metadata_models = get_metadata_model_for_data_type(kiara=kiara, data_type=type_cls._data_type_name) if kiara is not None: qual_profiles = kiara.type_registry.get_associated_profiles(type_cls._data_type_name) # type: ignore @@ -860,7 +943,7 @@ def category_name(cls) -> str: qualifier_profiles: Union[Mapping[str, Mapping[str, Any]], None] = Field( description="A map of qualifier profiles for this data types." ) - supported_properties: "MetadataTypeClassesInfo" = Field(description="The supported property types for this data type.") + supported_properties: MetadataTypeClassesInfo = Field(description="The supported property types for this data type.") _kiara: Union["Kiara", None] = PrivateAttr(default=None) def _retrieve_id(self) -> str: @@ -872,28 +955,32 @@ def _retrieve_data_to_hash(self) -> Any: def create_renderable(self, **config: Any) -> RenderableType: include_doc = config.get("include_doc", True) + include_lineage = config.get("include_lineage", True) + include_qualifer_profiles = config.get("include_qualifier_profiles", True) table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) table.add_column("property", style="i") table.add_column("value") - if self.lineage: - table.add_row("lineage", "\n".join(self.lineage[0:])) - else: - table.add_row("lineage", "-- n/a --") - - if self.qualifier_profiles: - qual_table = Table(show_header=False, box=box.SIMPLE) - qual_table.add_column("name") - qual_table.add_column("config") - for name, details in self.qualifier_profiles.items(): - json_details = orjson_dumps(details, option=orjson.OPT_INDENT_2) - qual_table.add_row( - name, Syntax(json_details, "json", background_color="default") - ) - table.add_row("qualifier profile(s)", qual_table) - else: - table.add_row("qualifier profile(s)", "-- n/a --") + if include_lineage: + if self.lineage: + table.add_row("lineage", "\n".join(self.lineage[0:])) + else: + table.add_row("lineage", "-- n/a --") + + if include_qualifer_profiles: + if self.qualifier_profiles: + qual_table = Table(show_header=False, box=box.SIMPLE) + qual_table.add_column("name") + qual_table.add_column("config") + for name, details in self.qualifier_profiles.items(): + json_details = orjson_dumps(details, option=orjson.OPT_INDENT_2) + qual_table.add_row( + name, Syntax(json_details, "json", background_color="default") + ) + table.add_row("qualifier profile(s)", qual_table) + else: + table.add_row("qualifier profile(s)", "-- n/a --") if include_doc: table.add_row( @@ -956,7 +1043,7 @@ def base_info_class(cls) -> Type[DataTypeClassInfo]: def create_renderable(self, **config: Any) -> RenderableType: full_doc = config.get("full_doc", False) - show_subtypes_inline = config.get("show_qualifier_profiles_inline", True) + show_subtypes_inline = config.get("show_qualifier_profiles_inline", False) show_lineage = config.get("show_type_lineage", True) show_lines = full_doc or show_subtypes_inline or show_lineage @@ -981,28 +1068,20 @@ def create_renderable(self, **config: Any) -> RenderableType: t_md = self.item_infos[type_name] # type: ignore row: List[Any] = [type_name] - if show_lineage: - if self._kiara is None: + lineage = t_md.lineage + if lineage is None: lineage_str = "-- n/a --" else: - lineage = list( - self._kiara.type_registry.get_type_lineage(type_name) - ) lineage_str = ", ".join(reversed(lineage[1:])) row.append(lineage_str) if show_subtypes_inline: - if self._kiara is None: - qual_profiles = "-- n/a --" + qual_profiles = t_md.qualifier_profiles + if not qual_profiles: + qual_profiles_str = "-- n/a --" else: - qual_p = self._kiara.type_registry.get_associated_profiles( - data_type_name=type_name - ).keys() - if qual_p: - qual_profiles = "\n".join(qual_p) - else: - qual_profiles = "-- n/a --" - row.append(qual_profiles) + qual_profiles_str = "\n".join(qual_profiles) + row.append(qual_profiles_str) if full_doc: md = Markdown(t_md.documentation.full_doc) @@ -2149,3 +2228,6 @@ def create_renderable(self, **config: Any) -> RenderableType: table.add_row(*row) return table + + + diff --git a/src/kiara/models/values/value_metadata/__init__.py b/src/kiara/models/values/value_metadata/__init__.py index d1254b2ff..0371b6b7b 100644 --- a/src/kiara/models/values/value_metadata/__init__.py +++ b/src/kiara/models/values/value_metadata/__init__.py @@ -9,37 +9,16 @@ from typing import ( TYPE_CHECKING, Any, - ClassVar, Dict, Iterable, - Literal, - Mapping, - Type, Union, ) -import orjson -from pydantic import Field -from rich import box -from rich.console import RenderableType -from rich.panel import Panel -from rich.syntax import Syntax -from rich.table import Table - from kiara.models import KiaraModel -from kiara.models.documentation import ( - AuthorsMetadataModel, - ContextMetadataModel, - DocumentationMetadataModel, -) -from kiara.interfaces.python_api.models.info import TypeInfo, TypeInfoItemGroup # from kiara.models.info import TypeInfo -from kiara.models.python_class import PythonClass -from kiara.utils.json import orjson_dumps if TYPE_CHECKING: - from kiara.context import Kiara from kiara.models.values.value import Value @@ -67,85 +46,3 @@ def _retrieve_data_to_hash(self) -> Any: return {"metadata": self.model_dump(), "schema": self.schema_json()} -class MetadataTypeInfo(TypeInfo): - - _kiara_model_id: ClassVar = "info.metadata_type" - - @classmethod - def create_from_type_class( - self, type_cls: Type[ValueMetadata], kiara: "Kiara" - ) -> "MetadataTypeInfo": - - authors_md = AuthorsMetadataModel.from_class(type_cls) - doc = DocumentationMetadataModel.from_class_doc(type_cls) - python_class = PythonClass.from_class(type_cls) - properties_md = ContextMetadataModel.from_class(type_cls) - type_name = type_cls._metadata_key # type: ignore - schema = type_cls.model_json_schema() - - return MetadataTypeInfo( - type_name=type_name, - documentation=doc, - authors=authors_md, - context=properties_md, - python_class=python_class, - metadata_schema=schema, - ) - - @classmethod - def base_class(self) -> Type[ValueMetadata]: - return ValueMetadata - - @classmethod - def category_name(cls) -> str: - return "value_metadata" - - metadata_schema: Dict[str, Any] = Field( - description="The (json) schema for this metadata value." - ) - - def create_renderable(self, **config: Any) -> RenderableType: - - include_doc = config.get("include_doc", True) - include_schema = config.get("include_schema", True) - - table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) - table.add_column("property", style="i") - table.add_column("value") - - if include_doc: - table.add_row( - "Documentation", - Panel(self.documentation.create_renderable(), box=box.SIMPLE), - ) - table.add_row("Author(s)", self.authors.create_renderable()) - table.add_row("Context", self.context.create_renderable()) - - if hasattr(self, "python_class"): - table.add_row("Python class", self.python_class.create_renderable()) - - if include_schema: - schema = Syntax( - orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2), - "json", - background_color="default", - ) - table.add_row("metadata_schema", schema) - - return table - - -class MetadataTypeClassesInfo(TypeInfoItemGroup): - - _kiara_model_id: ClassVar = "info.metadata_types" - - @classmethod - def base_info_class(cls) -> Type[TypeInfo]: - return MetadataTypeInfo - - type_name: Literal["value_metadata"] = "value_metadata" - item_infos: Mapping[str, MetadataTypeInfo] = Field( # type: ignore - description="The value metadata info instances for each type." - ) - - diff --git a/src/kiara/utils/metadata.py b/src/kiara/utils/metadata.py index 284ea9a5e..51c6be927 100644 --- a/src/kiara/utils/metadata.py +++ b/src/kiara/utils/metadata.py @@ -1,26 +1,34 @@ # -*- coding: utf-8 -*- - +from functools import lru_cache # Copyright (c) 2021, University of Luxembourg / DHARPA project # Copyright (c) 2021, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import Dict, Type, Union +from typing import Dict, Type, Union, Mapping, TYPE_CHECKING -from kiara.models.values.value_metadata import MetadataTypeClassesInfo, ValueMetadata +from kiara.models.values.value_metadata import ValueMetadata from kiara.registries.models import ModelRegistry +if TYPE_CHECKING: + from kiara.context import Kiara + from kiara.interfaces.python_api.models.info import MetadataTypeClassesInfo, MetadataTypeInfo + +@lru_cache() def find_metadata_models( alias: Union[str, None] = None, only_for_package: Union[str, None] = None -) -> MetadataTypeClassesInfo: +) -> "MetadataTypeClassesInfo": + + from kiara.interfaces.python_api.models.info import MetadataTypeClassesInfo model_registry = ModelRegistry.instance() _group = model_registry.get_models_of_type(ValueMetadata) # type: ignore classes: Dict[str, Type[ValueMetadata]] = {} for model_id, info in _group.item_infos.items(): - classes[model_id] = info.python_class.get_class() # type: ignore + model_cls = info.python_class.get_class() # type: ignore + classes[model_id] = model_cls group: MetadataTypeClassesInfo = MetadataTypeClassesInfo.create_from_type_items(group_title=alias, kiara=None, **classes) # type: ignore @@ -35,3 +43,39 @@ def find_metadata_models( ) return group + + +def get_metadata_model_for_data_type(kiara: "Kiara", data_type: str) -> "MetadataTypeClassesInfo": + """ + Return all available metadata extract operations for the provided type (and it's parent types). + + Arguments: + --------- + data_type: the value type + + Returns: + ------- + a mapping with the metadata type as key, and the operation as value + """ + + from kiara.interfaces.python_api.models.info import MetadataTypeClassesInfo + + lineage = set( + kiara.type_registry.get_type_lineage(data_type_name=data_type) + ) + + model_registry = ModelRegistry.instance() + all_metadata_models = model_registry.get_models_of_type(ValueMetadata) + + matching_types = {} + + for name, model_info in all_metadata_models.item_infos.items(): + + metadata_cls: Type[ValueMetadata] = model_info.python_class.get_class() + supported = metadata_cls.retrieve_supported_data_types() + if data_type in supported: + matching_types[name] = metadata_cls + + result = MetadataTypeClassesInfo.create_from_type_items(kiara=kiara, group_title=f"Metadata models for type '{data_type}'", **matching_types) + + return result