From b7e543fade401abf68b2fe4e65238927c07195b8 Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Mon, 12 Feb 2024 13:27:12 +0100 Subject: [PATCH] refactor: context option sqlite --- src/kiara/context/config.py | 145 +++++++++++++++---- src/kiara/interfaces/__init__.py | 1 + src/kiara/interfaces/cli/context/commands.py | 1 - src/kiara/interfaces/python_api/__init__.py | 46 +++++- 4 files changed, 160 insertions(+), 33 deletions(-) diff --git a/src/kiara/context/config.py b/src/kiara/context/config.py index 75d743fc0..7be36c34c 100644 --- a/src/kiara/context/config.py +++ b/src/kiara/context/config.py @@ -223,6 +223,82 @@ def archives(self) -> List["KiaraArchive"]: class KiaraContextConfig(BaseModel): + @classmethod + def create_from_sqlite_db(cls, db_path: Path) -> "KiaraContextConfig": + + import sqlite3 + + if not db_path.exists(): + context_id = str(uuid.uuid4()) + conn = sqlite3.connect(db_path) + c = conn.cursor() + c.execute( + """CREATE TABLE context_metadata + (key text PRIMARY KEY , value text NOT NULL)""" + ) + c.execute( + "INSERT INTO context_metadata VALUES ('context_id', ?)", (context_id,) + ) + c.execute( + """CREATE TABLE archive_metadata + (key text PRIMARY KEY , value text NOT NULL)""" + ) + c.execute( + "INSERT INTO archive_metadata VALUES ('archive_id', ?)", (context_id,) + ) + + conn.commit() + conn.close() + else: + try: + + with sqlite3.connect(db_path) as conn: + context_id = conn.execute( + "SELECT value FROM context_metadata WHERE key = 'context_id'" + ).fetchone()[0] + except Exception as e: + raise KiaraException( + f"Can't read context from sqlite db '{db_path}': {e}" + ) + + base_path = os.path.abspath(kiara_app_dirs.user_data_dir) + stores_base_path = os.path.join(base_path, "stores") + workflow_base_path = os.path.join( + stores_base_path, "filesystem_stores", "workflows" + ) + workflow_store_path = os.path.join(workflow_base_path, context_id) + + data_store_config = KiaraArchiveConfig( + archive_type="sqlite_data_store", + config={"sqlite_db_path": db_path.as_posix()}, + ) + alias_store_config = KiaraArchiveConfig( + archive_type="sqlite_alias_store", + config={"sqlite_db_path": db_path.as_posix()}, + ) + job_store_config = KiaraArchiveConfig( + archive_type="sqlite_job_store", + config={"sqlite_db_path": db_path.as_posix()}, + ) + workflow_store_config = KiaraArchiveConfig( + archive_type="filesystem_workflow_store", + config={"archive_path": workflow_store_path}, + ) + + archives = { + DEFAULT_DATA_STORE_MARKER: data_store_config, + DEFAULT_ALIAS_STORE_MARKER: alias_store_config, + DEFAULT_JOB_STORE_MARKER: job_store_config, + DEFAULT_WORKFLOW_STORE_MARKER: workflow_store_config, + } + + context_config = cls( + context_id=context_id, + archives=archives, + ) + + return context_config + model_config = ConfigDict(extra="forbid") context_id: str = Field(description="A globally unique id for this kiara context.") @@ -515,7 +591,7 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: def create_default_sqlite_archive_config() -> Dict[str, Any]: store_id = str(uuid.uuid4()) - file_name = f"{store_id}.sqlite" + file_name = f"{store_id}.karchive" archive_path = Path( os.path.abspath(os.path.join(sqlite_base_path, file_name)) ) @@ -638,39 +714,51 @@ def create_context_config( if not context_alias: context_alias = DEFAULT_CONTEXT_NAME + if context_alias in self.available_context_names: raise Exception( f"Can't create kiara context '{context_alias}': context with that alias already registered." ) - if os.path.sep in context_alias: - raise Exception( - f"Can't create context with alias '{context_alias}': no special characters allowed." + if context_alias.endswith(".kontext"): + context_db_file = Path(context_alias) + context_config: KiaraContextConfig = ( + KiaraContextConfig.create_from_sqlite_db(db_path=context_db_file) ) + self._validate_context(context_config=context_config) + context_config._context_config_path = context_db_file + else: - context_file = ( - Path(os.path.join(self.context_search_paths[0])) / f"{context_alias}.yaml" - ) + if os.path.sep in context_alias: + raise Exception( + f"Can't create context with alias '{context_alias}': no special characters allowed." + ) - archives: Dict[str, KiaraArchiveConfig] = {} - # create_default_archives(kiara_config=self) - context_id = ID_REGISTRY.generate( - obj_type=KiaraContextConfig, comment=f"new kiara context '{context_alias}'" - ) + context_file = ( + Path(os.path.join(self.context_search_paths[0])) + / f"{context_alias}.yaml" + ) - context_config = KiaraContextConfig( - context_id=str(context_id), archives=archives, extra_pipelines=[] - ) + archives: Dict[str, KiaraArchiveConfig] = {} + # create_default_archives(kiara_config=self) + context_id = ID_REGISTRY.generate( + obj_type=KiaraContextConfig, + comment=f"new kiara context '{context_alias}'", + ) - self._validate_context(context_config=context_config) + context_config = KiaraContextConfig( + context_id=str(context_id), archives=archives, extra_pipelines=[] + ) + + self._validate_context(context_config=context_config) - context_file.parent.mkdir(parents=True, exist_ok=True) - with open(context_file, "wt") as f: - yaml.dump(context_config.model_dump(), f) + context_file.parent.mkdir(parents=True, exist_ok=True) + with open(context_file, "wt") as f: + yaml.dump(context_config.model_dump(), f) - context_config._context_config_path = context_file + context_config._context_config_path = context_file + self._available_context_files[context_alias] = context_file - self._available_context_files[context_alias] = context_file self._context_data[context_alias] = context_config return context_config @@ -687,13 +775,20 @@ def create_context( with contextlib.suppress(Exception): context = uuid.UUID(context) # type: ignore - if isinstance(context, str) and os.path.exists(context): + if isinstance(context, str) and ( + os.path.exists(context) or context.endswith(".kontext") + ): context = Path(context) if isinstance(context, Path): - with context.open("rt") as f: - data = yaml.load(f) - context_config = KiaraContextConfig(**data) + if context.name.endswith(".kontext"): + context_config = KiaraContextConfig.create_from_sqlite_db( + db_path=context + ) + else: + with context.open("rt") as f: + data = yaml.load(f) + context_config = KiaraContextConfig(**data) elif isinstance(context, str): context_config = self.get_context_config(context_name=context) elif isinstance(context, uuid.UUID): diff --git a/src/kiara/interfaces/__init__.py b/src/kiara/interfaces/__init__.py index b944c217f..68e71911b 100644 --- a/src/kiara/interfaces/__init__.py +++ b/src/kiara/interfaces/__init__.py @@ -230,6 +230,7 @@ def __init__( ensure_plugins: Union[str, Iterable[str], None] = None, exit_process: bool = True, ): + if not context: context = os.environ.get("KIARA_CONTEXT", None) diff --git a/src/kiara/interfaces/cli/context/commands.py b/src/kiara/interfaces/cli/context/commands.py index b996f9880..ad6ae9470 100644 --- a/src/kiara/interfaces/cli/context/commands.py +++ b/src/kiara/interfaces/cli/context/commands.py @@ -87,7 +87,6 @@ def explain_context( if len(contexts) == 1: kcc = kiara_config.get_context_config(contexts[0]) - cs = ContextInfo.create_from_context_config( kcc, context_name=contexts[0], runtime_config=kiara_config.runtime_config ) diff --git a/src/kiara/interfaces/python_api/__init__.py b/src/kiara/interfaces/python_api/__init__.py index 835ad0e71..28bc215f7 100644 --- a/src/kiara/interfaces/python_api/__init__.py +++ b/src/kiara/interfaces/python_api/__init__.py @@ -198,6 +198,9 @@ def retrieve_plugin_info(self, plugin_name: str) -> KiaraPluginInfo: """ Get information about a plugin. + This contains information about included data-types, modules, operations, pipelines, as well as metadata + about author(s), etc. + Arguments: plugin_name: the name of the plugin @@ -211,6 +214,10 @@ def retrieve_plugin_info(self, plugin_name: str) -> KiaraPluginInfo: return info def retrieve_plugin_infos(self, plugin_name_regex: str = "^kiara[-_]plugin\\..*"): + """Get information about multiple plugins. + + This is just a convenience method to get information about multiple plugins at once. + """ if not plugin_name_regex: plugin_name_regex = "^kiara[-_]plugin\\..*" @@ -236,11 +243,18 @@ def context(self) -> "Kiara": return self._current_context def get_runtime_config(self) -> "KiaraRuntimeConfig": - """Retrieve the current runtime configuration.""" + """Retrieve the current runtime configuration. + + Check the 'KiaraRuntimeConfig' class for more information about the available options. + """ return self.context.runtime_config def get_context_info(self) -> ContextInfo: - """Retrieve information about the current kiara context.""" + """Retrieve information about the current kiara context. + + This contains information about the context, like its name/alias, the values & aliases it contains, and which archives are connected to it. + + """ context_config = self._kiara_config.get_context_config( self.get_current_context_name() ) @@ -258,7 +272,8 @@ def ensure_plugin_packages( """ Ensure that the specified packages are installed. - This functionality is provisional, don't rely on it being available long-term. Ideally, we'll have other, external ways to manage the environment. + + NOTE: this is not tested, and it might go away in the future, so don't rely on it being available long-term. Ideally, we'll have other, external ways to manage the environment. Arguments: package_names: The names of the packages to install. @@ -364,23 +379,35 @@ def __getattribute__(self, item): # ================================================================================================================== # context-management related functions def list_context_names(self) -> List[str]: - """list the names of all available/registered contexts.""" + """list the names of all available/registered contexts. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and + whether we want to support single-file contexts in the future. + """ return list(self._kiara_config.available_context_names) def retrieve_context_infos(self) -> ContextInfos: - """Retrieve information about the available/registered contexts.""" + """Retrieve information about the available/registered contexts. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ return ContextInfos.create_context_infos(self._kiara_config.context_configs) def get_current_context_name(self) -> str: - """Retrieve the name fo the current context.""" + """Retrieve the name of the current context. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ if self._current_context_alias is None: self.context return self._current_context_alias # type: ignore - def create_new_context(self, context_name: str, set_active: bool) -> None: + def create_new_context(self, context_name: str, set_active: bool = True) -> None: """ Create a new context. + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + Arguments: context_name: the name of the new context set_active: set the newly created context as the active one @@ -396,6 +423,10 @@ def create_new_context(self, context_name: str, set_active: bool) -> None: self._current_context_alias = context_name def set_active_context(self, context_name: str, create: bool = False) -> None: + """Set the currently active context for this KiarAPI instance. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ if not context_name: raise Exception("No context name provided.") @@ -433,6 +464,7 @@ def list_data_type_names(self, include_profiles: bool = False) -> List[str]: def is_internal_data_type(self, data_type_name: str) -> bool: """Checks if the data type is prepdominantly used internally by kiara, or whether it should be exposed to the user.""" + return self.context.type_registry.is_internal_type( data_type_name=data_type_name )