refactor: context option sqlite

src/kiara/context/config.py

@@ -223,6 +223,82 @@ def archives(self) -> List["KiaraArchive"]:
 
 
 class KiaraContextConfig(BaseModel):
+    @classmethod
+    def create_from_sqlite_db(cls, db_path: Path) -> "KiaraContextConfig":
+
+        import sqlite3
+
+        if not db_path.exists():
+            context_id = str(uuid.uuid4())
+            conn = sqlite3.connect(db_path)
+            c = conn.cursor()
+            c.execute(
+                """CREATE TABLE context_metadata
+                         (key text PRIMARY KEY , value text NOT NULL)"""
+            )
+            c.execute(
+                "INSERT INTO context_metadata VALUES ('context_id', ?)", (context_id,)
+            )
+            c.execute(
+                """CREATE TABLE archive_metadata
+                         (key text PRIMARY KEY , value text NOT NULL)"""
+            )
+            c.execute(
+                "INSERT INTO archive_metadata VALUES ('archive_id', ?)", (context_id,)
+            )
+
+            conn.commit()
+            conn.close()
+        else:
+            try:
+
+                with sqlite3.connect(db_path) as conn:
+                    context_id = conn.execute(
+                        "SELECT value FROM context_metadata WHERE key = 'context_id'"
+                    ).fetchone()[0]
+            except Exception as e:
+                raise KiaraException(
+                    f"Can't read context from sqlite db '{db_path}': {e}"
+                )
+
+        base_path = os.path.abspath(kiara_app_dirs.user_data_dir)
+        stores_base_path = os.path.join(base_path, "stores")
+        workflow_base_path = os.path.join(
+            stores_base_path, "filesystem_stores", "workflows"
+        )
+        workflow_store_path = os.path.join(workflow_base_path, context_id)
+
+        data_store_config = KiaraArchiveConfig(
+            archive_type="sqlite_data_store",
+            config={"sqlite_db_path": db_path.as_posix()},
+        )
+        alias_store_config = KiaraArchiveConfig(
+            archive_type="sqlite_alias_store",
+            config={"sqlite_db_path": db_path.as_posix()},
+        )
+        job_store_config = KiaraArchiveConfig(
+            archive_type="sqlite_job_store",
+            config={"sqlite_db_path": db_path.as_posix()},
+        )
+        workflow_store_config = KiaraArchiveConfig(
+            archive_type="filesystem_workflow_store",
+            config={"archive_path": workflow_store_path},
+        )
+
+        archives = {
+            DEFAULT_DATA_STORE_MARKER: data_store_config,
+            DEFAULT_ALIAS_STORE_MARKER: alias_store_config,
+            DEFAULT_JOB_STORE_MARKER: job_store_config,
+            DEFAULT_WORKFLOW_STORE_MARKER: workflow_store_config,
+        }
+
+        context_config = cls(
+            context_id=context_id,
+            archives=archives,
+        )
+
+        return context_config
+
     model_config = ConfigDict(extra="forbid")
 
     context_id: str = Field(description="A globally unique id for this kiara context.")
@@ -515,7 +591,7 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool:
         def create_default_sqlite_archive_config() -> Dict[str, Any]:
 
             store_id = str(uuid.uuid4())
-            file_name = f"{store_id}.sqlite"
+            file_name = f"{store_id}.karchive"
             archive_path = Path(
                 os.path.abspath(os.path.join(sqlite_base_path, file_name))
             )
@@ -638,39 +714,51 @@ def create_context_config(
 
         if not context_alias:
             context_alias = DEFAULT_CONTEXT_NAME
+
         if context_alias in self.available_context_names:
             raise Exception(
                 f"Can't create kiara context '{context_alias}': context with that alias already registered."
             )
 
-        if os.path.sep in context_alias:
-            raise Exception(
-                f"Can't create context with alias '{context_alias}': no special characters allowed."
+        if context_alias.endswith(".kontext"):
+            context_db_file = Path(context_alias)
+            context_config: KiaraContextConfig = (
+                KiaraContextConfig.create_from_sqlite_db(db_path=context_db_file)
             )
+            self._validate_context(context_config=context_config)
+            context_config._context_config_path = context_db_file
+        else:
 
-        context_file = (
-            Path(os.path.join(self.context_search_paths[0])) / f"{context_alias}.yaml"
-        )
+            if os.path.sep in context_alias:
+                raise Exception(
+                    f"Can't create context with alias '{context_alias}': no special characters allowed."
+                )
 
-        archives: Dict[str, KiaraArchiveConfig] = {}
-        # create_default_archives(kiara_config=self)
-        context_id = ID_REGISTRY.generate(
-            obj_type=KiaraContextConfig, comment=f"new kiara context '{context_alias}'"
-        )
+            context_file = (
+                Path(os.path.join(self.context_search_paths[0]))
+                / f"{context_alias}.yaml"
+            )
 
-        context_config = KiaraContextConfig(
-            context_id=str(context_id), archives=archives, extra_pipelines=[]
-        )
+            archives: Dict[str, KiaraArchiveConfig] = {}
+            # create_default_archives(kiara_config=self)
+            context_id = ID_REGISTRY.generate(
+                obj_type=KiaraContextConfig,
+                comment=f"new kiara context '{context_alias}'",
+            )
 
-        self._validate_context(context_config=context_config)
+            context_config = KiaraContextConfig(
+                context_id=str(context_id), archives=archives, extra_pipelines=[]
+            )
+
+            self._validate_context(context_config=context_config)
 
-        context_file.parent.mkdir(parents=True, exist_ok=True)
-        with open(context_file, "wt") as f:
-            yaml.dump(context_config.model_dump(), f)
+            context_file.parent.mkdir(parents=True, exist_ok=True)
+            with open(context_file, "wt") as f:
+                yaml.dump(context_config.model_dump(), f)
 
-        context_config._context_config_path = context_file
+            context_config._context_config_path = context_file
+            self._available_context_files[context_alias] = context_file
 
-        self._available_context_files[context_alias] = context_file
         self._context_data[context_alias] = context_config
 
         return context_config
@@ -687,13 +775,20 @@ def create_context(
             with contextlib.suppress(Exception):
                 context = uuid.UUID(context)  # type: ignore
 
-        if isinstance(context, str) and os.path.exists(context):
+        if isinstance(context, str) and (
+            os.path.exists(context) or context.endswith(".kontext")
+        ):
             context = Path(context)
 
         if isinstance(context, Path):
-            with context.open("rt") as f:
-                data = yaml.load(f)
-            context_config = KiaraContextConfig(**data)
+            if context.name.endswith(".kontext"):
+                context_config = KiaraContextConfig.create_from_sqlite_db(
+                    db_path=context
+                )
+            else:
+                with context.open("rt") as f:
+                    data = yaml.load(f)
+                context_config = KiaraContextConfig(**data)
         elif isinstance(context, str):
             context_config = self.get_context_config(context_name=context)
         elif isinstance(context, uuid.UUID):

src/kiara/interfaces/__init__.py

@@ -230,6 +230,7 @@ def __init__(
         ensure_plugins: Union[str, Iterable[str], None] = None,
         exit_process: bool = True,
     ):
+
         if not context:
             context = os.environ.get("KIARA_CONTEXT", None)
 

src/kiara/interfaces/cli/context/commands.py

@@ -87,7 +87,6 @@ def explain_context(
     if len(contexts) == 1:
 
         kcc = kiara_config.get_context_config(contexts[0])
-
         cs = ContextInfo.create_from_context_config(
             kcc, context_name=contexts[0], runtime_config=kiara_config.runtime_config
         )

src/kiara/interfaces/python_api/__init__.py

@@ -198,6 +198,9 @@ def retrieve_plugin_info(self, plugin_name: str) -> KiaraPluginInfo:
         """
         Get information about a plugin.
 
+        This contains information about included data-types, modules, operations, pipelines, as well as metadata
+        about author(s), etc.
+
         Arguments:
             plugin_name: the name of the plugin
 
@@ -211,6 +214,10 @@ def retrieve_plugin_info(self, plugin_name: str) -> KiaraPluginInfo:
         return info
 
     def retrieve_plugin_infos(self, plugin_name_regex: str = "^kiara[-_]plugin\\..*"):
+        """Get information about multiple plugins.
+
+        This is just a convenience method to get information about multiple plugins at once.
+        """
 
         if not plugin_name_regex:
             plugin_name_regex = "^kiara[-_]plugin\\..*"
@@ -236,11 +243,18 @@ def context(self) -> "Kiara":
         return self._current_context
 
     def get_runtime_config(self) -> "KiaraRuntimeConfig":
-        """Retrieve the current runtime configuration."""
+        """Retrieve the current runtime configuration.
+
+        Check the 'KiaraRuntimeConfig' class for more information about the available options.
+        """
         return self.context.runtime_config
 
     def get_context_info(self) -> ContextInfo:
-        """Retrieve information about the current kiara context."""
+        """Retrieve information about the current kiara context.
+
+        This contains information about the context, like its name/alias, the values & aliases it contains, and which archives are connected to it.
+
+        """
         context_config = self._kiara_config.get_context_config(
             self.get_current_context_name()
         )
@@ -258,7 +272,8 @@ def ensure_plugin_packages(
         """
         Ensure that the specified packages are installed.
 
-        This functionality is provisional, don't rely on it being available long-term. Ideally, we'll have other, external ways to manage the environment.
+
+        NOTE: this is not tested, and it might go away in the future, so don't rely on it being available long-term. Ideally, we'll have other, external ways to manage the environment.
 
         Arguments:
           package_names: The names of the packages to install.
@@ -364,23 +379,35 @@ def __getattribute__(self, item):
     # ==================================================================================================================
     # context-management related functions
     def list_context_names(self) -> List[str]:
-        """list the names of all available/registered contexts."""
+        """list the names of all available/registered contexts.
+
+        NOTE: this functionality might be changed in the future, depending on requirements and feedback and
+        whether we want to support single-file contexts in the future.
+        """
         return list(self._kiara_config.available_context_names)
 
     def retrieve_context_infos(self) -> ContextInfos:
-        """Retrieve information about the available/registered contexts."""
+        """Retrieve information about the available/registered contexts.
+
+        NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future.
+        """
         return ContextInfos.create_context_infos(self._kiara_config.context_configs)
 
     def get_current_context_name(self) -> str:
-        """Retrieve the name fo the current context."""
+        """Retrieve the name of the current context.
+
+        NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future.
+        """
         if self._current_context_alias is None:
             self.context
         return self._current_context_alias  # type: ignore
 
-    def create_new_context(self, context_name: str, set_active: bool) -> None:
+    def create_new_context(self, context_name: str, set_active: bool = True) -> None:
         """
         Create a new context.
 
+        NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future.
+
         Arguments:
             context_name: the name of the new context
             set_active: set the newly created context as the active one
@@ -396,6 +423,10 @@ def create_new_context(self, context_name: str, set_active: bool) -> None:
             self._current_context_alias = context_name
 
     def set_active_context(self, context_name: str, create: bool = False) -> None:
+        """Set the currently active context for this KiarAPI instance.
+
+        NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future.
+        """
 
         if not context_name:
             raise Exception("No context name provided.")
@@ -433,6 +464,7 @@ def list_data_type_names(self, include_profiles: bool = False) -> List[str]:
 
     def is_internal_data_type(self, data_type_name: str) -> bool:
         """Checks if the data type is prepdominantly used internally by kiara, or whether it should be exposed to the user."""
+
         return self.context.type_registry.is_internal_type(
             data_type_name=data_type_name
         )