From 099e47e5f37d2c35b956409082a24046dd232dc6 Mon Sep 17 00:00:00 2001 From: Eike Date: Fri, 10 Jan 2025 16:42:59 +0100 Subject: [PATCH] Update nix dev setup --- DEVELOPING.md | 6 +- flake.lock | 161 ++++++++------------------------------------------ flake.nix | 101 ++++++------------------------- 3 files changed, 44 insertions(+), 224 deletions(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index 55e7616b3..aee881889 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -96,8 +96,7 @@ environment can be created: environment. 2. In another terminal, run `vm-run` (headless) to start a vm running necessary external services, like the PostgreSQL database. -3. Potentially run `poetry-fix-cfg` to alter the `pyvenv.cfg` so that - poetry will use the env built by nix +3. Run `poetry install` to install the python venv Then `make run`, `make tests` etc can be used as usual. @@ -105,9 +104,6 @@ The environment also contains other useful tools, like ruff-lsp, pyright and more. Instead of a vm, a development environment using NixOS containers is also available. -The first invocation will take a while for the first run, as the -python environment is being built. Subsequent calls are then instant. - It will run a bash shell, check out [direnv](https://direnv.net/) and the [use flake](https://direnv.net/man/direnv-stdlib.1.html#codeuse-flake-ltinstallablegtcode) function if you prefer to keep your favorite shell. diff --git a/flake.lock b/flake.lock index 4973e05b3..01347bf31 100644 --- a/flake.lock +++ b/flake.lock @@ -6,11 +6,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1724320658, - "narHash": "sha256-YTgT2yHf936sxRHZiggOf5g+wcfxnlz9fAbRSKh5ukI=", + "lastModified": 1737104970, + "narHash": "sha256-WbKxxro4ZlVql5yDJpQMWc8SXsmzAQbEVo9RBnuYs8U=", "owner": "eikek", "repo": "devshell-tools", - "rev": "8f2b96bb183d647ca84a8538305f2b0529527b18", + "rev": "8dbbd035cff334476c9d46ddbb316c1f2ab7df3a", "type": "github" }, "original": { @@ -24,11 +24,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -42,11 +42,11 @@ "systems": "systems_2" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -55,52 +55,13 @@ "type": "github" } }, - "flake-utils_3": { - "inputs": { - "systems": "systems_3" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nix-github-actions": { - "inputs": { - "nixpkgs": [ - "poetry2nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1703863825, - "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", - "owner": "nix-community", - "repo": "nix-github-actions", - "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-github-actions", - "type": "github" - } - }, - "nixpkgs": { + "nix05": { "locked": { - "lastModified": 1721548954, - "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", + "lastModified": 1735563628, + "narHash": "sha256-OnSAY7XDSx7CtDoqNh8jwVwh4xNL/2HaJxGjryLWzX8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", + "rev": "b134951a4c9f3c995fd7be05f3243f8ecd65d798", "type": "github" }, "original": { @@ -110,66 +71,44 @@ "type": "github" } }, - "nixpkgs_2": { + "nixpkgs": { "locked": { - "lastModified": 1724224171, - "narHash": "sha256-zyKHydkh5PPQiDPKsgpUmtHGAjR5sNcCKAe56+3W4K0=", + "lastModified": 1736867362, + "narHash": "sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aea6e7b3614e103cdab310050a9046a4985886d6", + "rev": "9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.05", + "ref": "nixos-24.11", "repo": "nixpkgs", "type": "github" } }, - "nixpkgs_3": { + "nixpkgs_2": { "locked": { - "lastModified": 1719763542, - "narHash": "sha256-mXkOj9sJ0f69Nkc2dGGOWtof9d1YNY8Le/Hia3RN+8Q=", + "lastModified": 1736883708, + "narHash": "sha256-uQ+NQ0/xYU0N1CnXsa2zghgNaOPxWpMJXSUJJ9W7140=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e6cdd8a11b26b4d60593733106042141756b54a3", + "rev": "eb62e6aa39ea67e0b8018ba8ea077efe65807dc8", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable-small", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } }, - "poetry2nix": { - "inputs": { - "flake-utils": "flake-utils_3", - "nix-github-actions": "nix-github-actions", - "nixpkgs": "nixpkgs_3", - "systems": "systems_4", - "treefmt-nix": "treefmt-nix" - }, - "locked": { - "lastModified": 1724208502, - "narHash": "sha256-TCRcEPSfgAw/t7kClmlr23s591N06mQCrhzlAO7cyFw=", - "owner": "nix-community", - "repo": "poetry2nix", - "rev": "884b66152b0c625b8220b570a31dc7acc36749a3", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "poetry2nix", - "type": "github" - } - }, "root": { "inputs": { "devshell-tools": "devshell-tools", "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs_2", - "poetry2nix": "poetry2nix" + "nix05": "nix05", + "nixpkgs": "nixpkgs_2" } }, "systems": { @@ -201,56 +140,6 @@ "repo": "default", "type": "github" } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_4": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "id": "systems", - "type": "indirect" - } - }, - "treefmt-nix": { - "inputs": { - "nixpkgs": [ - "poetry2nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1719749022, - "narHash": "sha256-ddPKHcqaKCIFSFc/cvxS14goUhCOAwsM1PbMr0ZtHMg=", - "owner": "numtide", - "repo": "treefmt-nix", - "rev": "8df5ff62195d4e67e2264df0b7f5e8c9995fd0bd", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "treefmt-nix", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 183f9c1ad..94fb86a79 100644 --- a/flake.nix +++ b/flake.nix @@ -1,24 +1,25 @@ { inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nix05.url = "github:NixOS/nixpkgs/nixos-24.05"; flake-utils.url = "github:numtide/flake-utils"; devshell-tools.url = "github:eikek/devshell-tools"; - poetry2nix.url = "github:nix-community/poetry2nix"; }; outputs = inputs @ { self, nixpkgs, + nix05, flake-utils, - devshell-tools, - poetry2nix, + devshell-tools }: { nixosConfigurations = let services = { services.dev-postgres = { enable = true; - databases = ["renku"]; + databases = ["renku_test"]; + init-script = ./.devcontainer/generate_ulid_func.sql; pgweb = { enable = true; database = "renku"; @@ -53,116 +54,50 @@ } // flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + pkgs05 = nix05.legacyPackages.${system}; devshellToolsPkgs = devshell-tools.packages.${system}; - poetryLib = poetry2nix.lib.mkPoetry2Nix {inherit pkgs;}; - p2n-args = { - projectDir = ./.; - python = pkgs.python312; - editablePackageSources = { - bases = ./bases; - components = ./components; - }; - extraPackages = p: [ - p.ruff-lsp - ]; - overrides = let - add-setuptools = name: final: prev: - prev.${name}.overridePythonAttrs (old: {buildInputs = (old.buildInputs or []) ++ [prev.setuptools];}); - add-poetry = name: final: prev: - prev.${name}.overridePythonAttrs (old: {buildInputs = (old.buildInputs or []) ++ [prev.poetry];}); - in - poetryLib.defaultPoetryOverrides.extend - (final: prev: { - appier = add-setuptools "appier" final prev; - inflector = add-setuptools "inflector" final prev; - google-api = add-setuptools "google-api" final prev; - sanic-ext = add-setuptools "sanic-ext" final prev; - undictify = add-setuptools "undictify" final prev; - types-cffi = add-setuptools "types-cffi" final prev; - avro-preprocessor = add-setuptools "avro-preprocessor" final prev; - authzed = add-poetry "authzed" final prev; - dataclasses-avroschema = add-poetry "dataclasses-avroschema" final prev; - datamodel-code-generator = add-poetry "datamodel-code-generator" final prev; - kubernetes-asyncio = add-setuptools "kubernetes-asyncio" final prev; - prometheus-sanic = - prev.prometheus-sanic.overridePythonAttrs - ( - old: { - buildInputs = (old.buildInputs or []) ++ [prev.poetry prev.poetry-core]; - # fix the wrong dependency - # see https://github.com/nix-community/poetry2nix/issues/1694 - postPatch = '' - substituteInPlace pyproject.toml --replace "poetry.masonry" "poetry.core.masonry" - ''; - } - ); - }); - }; - - projectEnv = poetryLib.mkPoetryEnv p2n-args; - devSettings = { CORS_ALLOW_ALL_ORIGINS = "true"; DB_USER = "dev"; DB_NAME = "renku"; DB_PASSWORD = "dev"; + PGPASSWORD = "dev"; AUTHZ_DB_KEY = "dev"; AUTHZ_DB_NO_TLS_CONNECTION = "true"; AUTHZ_DB_GRPC_PORT = "50051"; ALEMBIC_CONFIG = "./components/renku_data_services/migrations/alembic.ini"; + NB_SERVER_OPTIONS__DEFAULTS_PATH = "server_defaults.json"; + NB_SERVER_OPTIONS__UI_CHOICES_PATH = "server_options.json"; - # necessary for poetry run … as it might need to compile stuff - # ONLY WHEN NOT using python from nix dev environment - #LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib"; + LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib"; POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON = "true"; POETRY_VIRTUALENVS_OPTIONS_SYSTEM_SITE_PACKAGES = "true"; + POETRY_INSTALLER_NO_BINARY = "ruff"; }; - fix-poetry-cfg = pkgs.writeShellScriptBin "poetry-fix-cfg" '' - python_exec="$(which python)" - python_bin="$(dirname "$python_exec")" - python_env="$(dirname "$python_bin")" - - env_path="$(poetry env info -p)" - if [ -z "$env_path" ]; then - poetry env use "$python_exec" - env_path="$(poetry env info -p)" - fi - env_cfg="$env_path/pyvenv.cfg" - - if [ ! -r "$env_path/pyvenv.cfg.bak" ]; then - cp "$env_path/pyvenv.cfg" "$env_path/pyvenv.cfg.bak" - fi - - echo "Fix paths in: $env_cfg" - ${pkgs.gnused}/bin/sed -i -E "s,home = (.*)$,home = $python_bin,g" "$env_cfg" - ${pkgs.gnused}/bin/sed -i -E "s,base-prefix = (.*)$,base-prefix = $python_env,g" "$env_cfg" - ${pkgs.gnused}/bin/sed -i -E "s,base-exec-prefix = (.*)$,base-exec-prefix = $python_env,g" "$env_cfg" - ${pkgs.gnused}/bin/sed -i -E "s,base-executable = (.*)$,base-executable = $python_exec,g" "$env_cfg" - ''; commonPackages = with pkgs; [ redis postgresql jq devshellToolsPkgs.openapi-docs spicedb - spicedb-zed + cargo + rustc + pkgs05.spicedb-zed ruff ruff-lsp poetry + python312 pyright - mypy rclone - fix-poetry-cfg ]; in { formatter = pkgs.alejandra; devShells = rec { default = vm; - vm = projectEnv.env.overrideAttrs (oldAttrs: - devSettings + vm = pkgs.mkShell (devSettings // { buildInputs = commonPackages @@ -183,7 +118,7 @@ ${pkgs.socat}/bin/socat TCP-LISTEN:50051,fork TCP:rsdevcnt:50051 ''; in - projectEnv.env.overrideAttrs (oldAttrs: + pkgs.mkShell ( devSettings // { buildInputs =