Release 0.3.29 (#1403)

# Description Prefix the PR title with the Jira issue number on the form `[CDF-12345]`. Please describe the change you have made. ## Checklist - [ ] Tests added/updated. - [ ] Run Demo Job Locally. - [ ] Documentation updated. - [ ] Changelogs updated in [CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md). - [ ] Template changelogs updated in [CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md). [CDF-12345]: https://cognitedata.atlassian.net/browse/CDF-12345?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
cognitedata · Jan 27, 2025 · 84d3e09 · 84d3e09
2 parents adb3b30 + a1022a5
commit 84d3e09
Show file tree

Hide file tree

Showing 45 changed files with 319 additions and 230 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -12,7 +12,7 @@ repos:
           - --fixable=E,W,F,I,T,RUF,TID,UP
           - --target-version=py39
       - id: ruff-format
-    rev: v0.9.2
+    rev: v0.9.3
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.43.0

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -29,11 +29,10 @@
       "request": "launch",
       "program": "./cdf-tk-dev.py",
       "args": [
-        "--verbose",
         "--override-env",
         "run",
         "function",
-        "live",
+        "local",
         "--env=local",
         "-o",
         "./cognite_toolkit/"
@@ -90,7 +89,9 @@
       "args": [
         "build",
         "--env=local",
-        "--verbose"
+        "--verbose",
+        "-o",
+        "./cognite_toolkit/"
       ],
       "console": "integratedTerminal",
       "justMyCode": false
@@ -120,77 +121,70 @@
       "justMyCode": false
     },
     {
-      "name": "Python: clean",
+      "name": "cdf modules pull interactive",
       "type": "debugpy",
       "request": "launch",
       "program": "./cdf-tk-dev.py",
       "args": [
-        "clean",
-        //"--dry-run",
-        "--env=local"
-        //"--include=transformations"
+        "modules",
+        "pull",
+        "--env", 
+        "local",
+        "-o",
+        "./cognite_toolkits/"
       ],
       "console": "integratedTerminal",
       "justMyCode": false
     },
     {
-      "name": "Python: auth verify",
+      "name": "cdf modules pull",
       "type": "debugpy",
       "request": "launch",
       "program": "./cdf-tk-dev.py",
-      "args": ["--override-env", "auth", "verify"],
+      "args": [
+        "modules",
+        "pull",
+        "--env", 
+        "local",
+        "-o",
+        "./cognite_toolkit/",
+        "hextractors"
+      ],
       "console": "integratedTerminal",
       "justMyCode": false
     },
     {
-      "name": "Python: cdf.py",
+      "name": "Python: clean",
       "type": "debugpy",
       "request": "launch",
       "program": "./cdf-tk-dev.py",
       "args": [
-        //"--project=TEST",
-        "run",
-        "function",
-        "--external_id=fn_test2",
-        "--local",
-        "--env=local",
-        "./cognite_toolkit/"
+        "modules",
+        "pull",
+        "--env", 
+        "local",
+        "-o",
+        "./cognite_toolkits/"
       ],
       "console": "integratedTerminal",
       "justMyCode": false
     },
     {
-      "name": "Python: P&ID annotation function test",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "./cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/handler.py",
-      "console": "integratedTerminal",
-      "justMyCode": false
-    },
-    {
-      "name": "Python: TS - Asset Contextualization function test",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "./cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/functions/fn_context_timeseries_oid_opcua_asset/handler.py",
-      "console": "integratedTerminal",
-      "justMyCode": false
-    },
-    {
-      "name": "Python: TS - Star of workflow test",
+      "name": "cdf run function local",
       "type": "debugpy",
       "request": "launch",
-      "program": "./cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_workflow_files_oid_fileshare_annotation/handler.py",
+      "program": "./cdf-tk-dev.py",
+      "args": [
+        "run",
+        "function",
+        "--env", 
+        "local",
+        "-o",
+        "./cognite_toolkit/"
+      ],
       "console": "integratedTerminal",
       "justMyCode": false
     },
-    {
-        "name": "Python: TS - 3D annotation function test",
-        "type": "debugpy",
-        "request": "launch",
-        "program": "./cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_3d_valhall/functions/fn_context_3d_oid_fileshare_asset/handler.py",
-        "console": "integratedTerminal",
-        "justMyCode": false
-    },
     {
       "name": "Python: init",
       "type": "debugpy",
@@ -209,15 +203,6 @@
       "console": "integratedTerminal",
       "justMyCode": false
     },
-    {
-      "name": "Python: build local",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "./cdf-tk-dev.py",
-      "args": ["build", "--env", "local", "./my_organization"],
-      "console": "integratedTerminal",
-      "justMyCode": false
-    },
     {
       "name": "Python: modules init",
       "type": "debugpy",
@@ -248,7 +233,9 @@
       "args": [
         "modules", 
         "list",
-        "--env", "local"],
+        "--env", "local", "-o",
+        "./cognite_toolkit/"
+      ],
       "console": "integratedTerminal",
       "justMyCode": false
     },

diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md
@@ -15,12 +15,27 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.29] - 2025-01-27
+
+### Added
+
+- Interactive selection mode if you don't provide a path or module name as argument to `cdf modules pull`.
+
+### Improved
+
+- Running `cdf deploy` and `cdf deploy --dry-run` now reports correctly delete + create for resources that do not
+  support update. This is `Function`, `FunctionSchedule`, `Group`, and `SequenceRows`.
+- [alpha feature] The `cdf purge space` now automatically reduces the batch size when deleting nodes when getting a
+  408 timeout error.
+- [alpha feature] The `cdf purge space` now tells which spaces needs to be purged first if the current space has
+  dependencies. This happens when a node in one space is used as a node type in another space.
+
 ## [0.3.28] - 2025-01-21
 
 ### Fixed
 
 - Variable replacement was too restrictive for environment variables. Previous behaviour to replace all
-  environment variables has been restored with the exception of Extraction Pipeline Configs.
+  environment variables has been restored except Extraction Pipeline Configs.
 - Supporting non-ascii characters like æ,ø, å, ç, ã in built files  
 
 ## [0.3.27] - 2025-01-20

diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md
@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.29] - 2025-01-27
+
+No changes to templates.
+
 ## [0.3.28] - 2025-01-21
 
 No changes to templates.

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -115,49 +115,47 @@ we are pushing the functionality into the Python SDK when that makes sense.
 
 ## Releasing
 
-The templates are bundled with the `cdf-tk` tool, so they are released together.
-To release a new version of the `cdf-tk` tool and the templates, you need to do the following:
+The templates are bundled with the `cdf` tool, so they are released together.
+To release a new version of the `cdf` tool and the templates, you need to do the following:
 
-1. Create a new preparation branch from `main` where you can make the final changes and do version bumping,
+1. Make sure that the CHANGELOG.cdf-tk.md is up to date. The top entry should be `## TBD` only.
+2. Go to <https://github.com/cognitedata/toolkit/actions/workflows/prepare-release.yaml> and run the workflow.
+
+   > Use `patch` to bump z in the x.y.z version number, `minor` to bump y, and `major` to bump x.
+
+   This will create a new preparation branch from `main` with final changes and version bumping,
    e.g. `prepare_for_0_1_0b3`. Use `aX` for alpha, `bX` for beta, and `rcX` for
    release candidate:
-   1. Update `CHANGELOG.cdf-tk.md` file with a header e.g. `## [0.1.0b3] - 2024-01-12` and review the
+   - Updates `CHANGELOG.cdf-tk.md` file with a header e.g. `## [0.1.0b3] - 2024-01-12` and review the
       change comments since the previous release. Ensure that the changes are correctly reflected in the
       comments and that the changes can be easily understood. Also verify that any breaking changes
       are clearly marked as such (`**BREAKING**`).
-   2. Do the same update to `CHANGELOG.templates.md` file.
-   3. Update the files with the new version number, this is done with
+   - Does the same update to `CHANGELOG.templates.md` file.
+   - Updates the files with the new version number, this is done with
       the `cdf bump --patch` (or `--minor`, `--major`, `--alpha`, `--beta`) command.
       - `cognite_toolkit/_version.py`
       - `pyproject.toml`
       - `_system.yaml` (multiple)
 
-      You can use the `python bump --minor --alpha` command to bump the version in all files.
-   4. Run `poetry lock` to update the `poetry.lock` file.
-   5. Run `pytest tests` locally to ensure that tests pass.
-   6. Run `python module_upgrade/run_check.py` to ensure that the `cdf-tk modules upgrade` command works as expected.
+   - Runs `poetry lock` to update the `poetry.lock` file.
+   - Runs `pytest tests` locally to ensure that tests pass.
+   - Runs `python module_upgrade/run_check.py` to ensure that the `cdf-tk modules upgrade` command works as expected
       against previous versions. See [Module Upgrade](module_upgrade/README.md) for more information.
 
-      if a check fails due to missing package:
-      - source .venv/.../bin/activate
-      - pip install dependency
-      - deactivate
-      - run script again
-
-1. Get approval to **squash merge** the branch into `main`:
+3. Get approval to **squash merge** the branch into `main`:
    1. Verify that all Github actions pass.
-1. Create a release branch: `release-x.y.z` from `main`:
+4. Create a release branch: `release-x.y.z` from `main`:
    1. Create a new tag on the branch with the version number, e.g. `v0.1.0b3`.
    2. Open a PR with the existing `release` branch as base comparing to your new `release-x.y.z` branch.
    3. Get approval and merge (**do not squash**).
    4. Verify that the Github action `release` passes and pushes to PyPi.
-1. Create a new release on github.com with the tag and release notes:
+5. Create a new release on github.com with the tag and release notes:
    1. Find the tag you created and create the new release.
    2. Copy the release notes from the `CHANGELOG.cdf-tk.md` file, add a `# cdf-tk` header.
    3. Copy then further below the release notes from the `CHANGELOG.templates.md` file, add
       a `# Templates` header.
    4. Remember to mark as pre-release if this is not a final release.
-1. Evaluate necessary announcements:
+6. Evaluate necessary announcements:
    1. On the Cognite Hub group, create a new post.
    2. As part of product releases, evaluate what to include.
    3. Cognite internal announcements.
diff --git a/cdf.toml b/cdf.toml
@@ -29,4 +29,4 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.28"
+version = "0.3.29"
diff --git a/cognite_toolkit/_builtin_modules/cdf.toml b/cognite_toolkit/_builtin_modules/cdf.toml
@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.28"
+version = "0.3.29"
 
 
 [plugins]

diff --git a/cognite_toolkit/_cdf_tk/apps/_modules_app.py b/cognite_toolkit/_cdf_tk/apps/_modules_app.py
@@ -124,13 +124,13 @@ def add(
     def pull(
         self,
         ctx: typer.Context,
-        module: Annotated[
-            str,
+        module_name_or_path: Annotated[
+            Optional[str],
             typer.Argument(
-                help="The module or path to module to pull from CDF.",
+                help="The module or path to module to pull from CDF. If not provided, interactive mode will be used.",
                 allow_dash=True,
             ),
-        ],
+        ] = None,
         organization_dir: Annotated[
             Path,
             typer.Option(
@@ -169,7 +169,7 @@ def pull(
         ToolGlobals = CDFToolConfig.from_context(ctx)
         cmd.run(
             lambda: cmd.pull_module(
-                module=module,
+                module_name_or_path=module_name_or_path,
                 organization_dir=organization_dir,
                 env=build_env,
                 dry_run=dry_run,