diff --git a/flake.lock b/flake.lock index 9613ccfb..ecfe7fd8 100644 --- a/flake.lock +++ b/flake.lock @@ -1,9 +1,46 @@ { "nodes": { + "devshell-tools": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1710232115, + "narHash": "sha256-IX9AA+FY38YK7UEmjRC53ylZiZ0DiLA+SAztUBGp2t4=", + "owner": "eikek", + "repo": "devshell-tools", + "rev": "5ea01c1e822642a5961cde0e5d19e84f9e46da40", + "type": "github" + }, + "original": { + "owner": "eikek", + "repo": "devshell-tools", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" }, + "locked": { + "lastModified": 1709126324, + "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "d465f4819400de7c8d874d50b982301f28a84605", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, "locked": { "lastModified": 1705309234, "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", @@ -19,6 +56,22 @@ } }, "nixpkgs": { + "locked": { + "lastModified": 1709309926, + "narHash": "sha256-VZFBtXGVD9LWTecGi6eXrE0hJ/mVB3zGUlHImUs2Qak=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "79baff8812a0d68e24a836df0a364c678089e2c7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1708447487, "narHash": "sha256-MGWA+gyB8xoYxzzMJKnZft2grvgTzN/86T9iGiNdUp4=", @@ -36,8 +89,9 @@ }, "root": { "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "devshell-tools": "devshell-tools", + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_2" } }, "systems": { @@ -54,6 +108,21 @@ "repo": "default", "type": "github" } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index a6c4e4a4..74cb7e3b 100644 --- a/flake.nix +++ b/flake.nix @@ -2,48 +2,27 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; flake-utils.url = "github:numtide/flake-utils"; + devshell-tools.url = "github:eikek/devshell-tools"; }; outputs = inputs @ { self, nixpkgs, flake-utils, + devshell-tools, }: { - overlays.default = final: prev: { - solr = self.packages.${prev.system}.solr; - openapi-doc = self.packages.${prev.system}.openapi-doc; - }; - nixosConfigurations = let - selfOverlay = { - lib, - config, - ... - }: { - nixpkgs.overlays = [ - self.overlays.default - ]; - system.stateVersion = "23.11"; - }; - in { - dev-vm = nixpkgs.lib.nixosSystem { + nixosConfigurations = { + rsdev-vm = devshell-tools.lib.mkVm { system = flake-utils.lib.system.x86_64-linux; - specialArgs = {inherit inputs;}; modules = [ - selfOverlay - ./nix/dev-vm.nix + ./nix/services.nix ]; }; - container = nixpkgs.lib.nixosSystem { + rsdev-cnt = devshell-tools.lib.mkContainer { system = flake-utils.lib.system.x86_64-linux; modules = [ - ({pkgs, ...}: { - boot.isContainer = true; - networking.useDHCP = false; - }) - selfOverlay - ./nix/solr-module.nix ./nix/services.nix ]; }; @@ -51,81 +30,79 @@ } // flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; - selfPkgs = self.packages.${system}; + selfPkgs = import ./nix/dev-scripts.nix { + inherit system devshell-tools; + inherit (pkgs) writeShellScriptBin; + }; + devshellToolsPkgs = devshell-tools.packages.${system}; + commonPackages = with pkgs; + [ + redis + jq + scala-cli + devshellToolsPkgs.sbt17 + devshellToolsPkgs.openapi-docs + ] + ++ (builtins.attrValues selfPkgs); + + queueNames = [ + "projectCreated" + "projectUpdated" + "projectRemoved" + "projectAuthorizationAdded" + "projectAuthorizationUpdated" + "projectAuthorizationRemoved" + "userAdded" + "userUpdated" + ]; + queueNameConfig = builtins.listToAttrs (builtins.map (qn: { + name = "RS_REDIS_QUEUE_${qn}"; + value = qn; + }) + queueNames); in { formatter = pkgs.alejandra; - packages = - ((import ./nix/dev-scripts.nix) {inherit (pkgs) concatTextFile writeShellScriptBin;}) - // rec { - solr = pkgs.callPackage (import ./nix/solr.nix) {}; - swagger-ui = pkgs.callPackage (import ./nix/swagger-ui.nix) {}; - openapi-doc = pkgs.callPackage (import ./nix/openapi-doc.nix) {inherit swagger-ui;}; - }; devShells = rec { default = container; - container = pkgs.mkShell { - RS_SOLR_HOST = "rsdev"; - RS_SOLR_URL = "http://rsdev:8983/solr"; - RS_SOLR_CORE = "rsdev-test"; - RS_REDIS_HOST = "rsdev"; - RS_REDIS_PORT = "6379"; - RS_CONTAINER = "rsdev"; - RS_LOG_LEVEL = "3"; - - #don't start docker container for dbTests - NO_SOLR = "true"; - NO_REDIS = "true"; + container = pkgs.mkShellNoCC (queueNameConfig + // { + RS_SOLR_HOST = "rsdev-cnt"; + RS_SOLR_URL = "http://rsdev-cnt:8983/solr"; + RS_SOLR_CORE = "rsdev-test"; + RS_REDIS_HOST = "rsdev-cnt"; + RS_REDIS_PORT = "6379"; + RS_CONTAINER = "rsdev"; + RS_LOG_LEVEL = "3"; - buildInputs = with pkgs; - with selfPkgs; [ - redis - jq - sbt - scala-cli + #don't start docker container for dbTests + NO_SOLR = "true"; + NO_REDIS = "true"; + DEV_CONTAINER = "rsdev-cnt"; - redis-push - recreate-container - start-container - solr-create-core - solr-delete-core - solr-recreate-core - solr-recreate-dbtests-cores - solr-logs - k8s-reprovision - ]; - }; - vm = pkgs.mkShell { - RS_SOLR_URL = "http://localhost:18983/solr"; - RS_SOLR_CORE = "rsdev-test"; - RS_REDIS_HOST = "localhost"; - RS_REDIS_PORT = "16379"; - VM_SSH_PORT = "10022"; - RS_LOG_LEVEL = "3"; + buildInputs = + commonPackages + ++ (builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts); + }); + vm = pkgs.mkShellNoCC (queueNameConfig + // { + RS_SOLR_URL = "http://localhost:18983/solr"; + RS_SOLR_CORE = "rsdev-test"; + RS_REDIS_HOST = "localhost"; + RS_REDIS_PORT = "16379"; + VM_SSH_PORT = "10022"; + RS_LOG_LEVEL = "3"; - #don't start docker container for dbTests - NO_SOLR = "true"; - NO_REDIS = "true"; + #don't start docker container for dbTests + NO_SOLR = "true"; + NO_REDIS = "true"; - buildInputs = with pkgs; - with selfPkgs; [ - redis - jq - sbt - scala-cli + DEV_VM = "rsdev-vm"; - redis-push - vm-build - vm-run - vm-ssh - vm-solr-create-core - vm-solr-delete-core - vm-solr-recreate-core - vm-solr-logs - solr-recreate-dbtests-cores - k8s-reprovision - ]; - }; + buildInputs = + commonPackages + ++ (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts); + }); }; }); } diff --git a/nix/dev-scripts.nix b/nix/dev-scripts.nix index 9fedaec4..ab6f4529 100644 --- a/nix/dev-scripts.nix +++ b/nix/dev-scripts.nix @@ -1,94 +1,31 @@ { - concatTextFile, + system, writeShellScriptBin, -}: let key = ./dev-vm-key; in rec { - redis-push = concatTextFile { - name = "redis-push"; - files = [./scripts/redis-push]; - executable = true; - destination = "/bin/redis-push"; + devshell-tools, +}: rec { + redis-push = devshell-tools.lib.installScript { + script = ./scripts/redis-push; + inherit system; }; - recreate-container = concatTextFile { - name = "recreate-container"; - files = [./scripts/recreate-container]; - executable = true; - destination = "/bin/recreate-container"; + k8s-reprovision = devshell-tools.lib.installScript { + script = ./scripts/k8s-reprovision; + inherit system; }; - start-container = writeShellScriptBin "start-container" '' - cnt=''${RS_CONTAINER:-rsdev} - sudo nixos-container start $cnt - ''; - - solr-create-core = writeShellScriptBin "solr-create-core" '' - core_name=''${1:-rsdev-test} - sudo nixos-container run ''${RS_CONTAINER:-rsdev} -- su solr -c "solr create -c $core_name" - sudo nixos-container run ''${RS_CONTAINER:-rsdev} -- find /var/solr/data/$core_name/conf -type f -exec chmod 644 {} \; - ''; - - solr-delete-core = writeShellScriptBin "solr-delete-core" '' - core_name=''${1:-rsdev-test} - sudo nixos-container run ''${RS_CONTAINER:-rsdev} -- su solr -c "solr delete -c $core_name" - ''; - - solr-recreate-core = writeShellScriptBin "solr-recreate-core" '' - ${solr-delete-core}/bin/solr-delete-core "$1" - ${solr-create-core}/bin/solr-create-core "$1" - ''; - - solr-logs = writeShellScriptBin "solr-logs" '' - sudo nixos-container run ''${RS_CONTAINER:-rsdev} -- journalctl -efu solr.service - ''; - - vm-build = writeShellScriptBin "vm-build" '' - nix build .#nixosConfigurations.dev-vm.config.system.build.vm - ''; - - vm-run = writeShellScriptBin "vm-run" '' - nix run .#nixosConfigurations.dev-vm.config.system.build.vm - ''; - - vm-ssh = writeShellScriptBin "vm-ssh" '' - ssh -i ${key} -p $VM_SSH_PORT root@localhost "$@" - ''; - - vm-solr-logs = writeShellScriptBin "solr-logs" '' - ${vm-ssh}/bin/vm-ssh journalctl -efu solr.service - ''; - - vm-solr-create-core = writeShellScriptBin "solr-create-core" '' - core_name=''${1:-rsdev-test} - ssh -p $VM_SSH_PORT root@localhost "su solr -c \"solr create -c $core_name\"" - ssh -p $VM_SSH_PORT root@localhost "find /var/solr/data/$core_name/conf -type f -exec chmod 644 {} \;" - ''; - - vm-solr-delete-core = writeShellScriptBin "solr-delete-core" '' - core_name=''${1:-rsdev-test} - ssh -p $VM_SSH_PORT root@localhost "su solr -c \"solr delete -c $core_name\"" - ''; - - vm-solr-recreate-core = writeShellScriptBin "solr-recreate-core" '' - ${vm-solr-delete-core}/bin/solr-delete-core "$1" - ${vm-solr-create-core}/bin/solr-create-core "$1" + solr-recreate-cores = writeShellScriptBin "solr-recreate-cores" '' + script=$([ "$(which cnt-solr-recreate-core)" != "" ] && echo "cnt-solr-recreate-core" || echo "vm-solr-recreate-core") + for c in $@; do + $script $c + done ''; # core names are defined in project/SolrServer.scala solr-recreate-dbtests-cores = writeShellScriptBin "solr-recreate-dbtests-cores" '' - solr-delete-core core-test1 - solr-delete-core core-test2 - solr-delete-core core-test3 - solr-delete-core search-core-test - solr-create-core core-test1 - solr-create-core core-test2 - solr-create-core core-test3 - solr-create-core search-core-test + ${solr-recreate-cores}/bin/solr-recreate-cores core-test1 core-test2 core-test3 search-core-test ''; - k8s-reprovision = concatTextFile { - name = "k8s-reprovision"; - files = [./scripts/k8s-reprovision]; - executable = true; - destination = "/bin/k8s-reprovision"; - }; + solr-recreate-search-core = writeShellScriptBin "solr-recreate-search-core" '' + ${solr-recreate-cores}/bin/solr-recreate-cores $RS_SOLR_CORE + ''; } diff --git a/nix/dev-vm-key b/nix/dev-vm-key deleted file mode 100644 index 518517a5..00000000 --- a/nix/dev-vm-key +++ /dev/null @@ -1,7 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW -QyNTUxOQAAACASODG0t0zVHAKwE/CSlmCpNlR/XiWBcsFXA9gDBMCfHwAAAJDN6ZQezemU -HgAAAAtzc2gtZWQyNTUxOQAAACASODG0t0zVHAKwE/CSlmCpNlR/XiWBcsFXA9gDBMCfHw -AAAEBoaSefL4ulXiGquSLqHHQ9rj+aZZ+YffV49VwEwrduBRI4MbS3TNUcArAT8JKWYKk2 -VH9eJYFywVcD2AMEwJ8fAAAACmVpa2VAcG9yb3MBAgM= ------END OPENSSH PRIVATE KEY----- diff --git a/nix/dev-vm-key.pub b/nix/dev-vm-key.pub deleted file mode 100644 index 13f1d2b4..00000000 --- a/nix/dev-vm-key.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBI4MbS3TNUcArAT8JKWYKk2VH9eJYFywVcD2AMEwJ8f diff --git a/nix/dev-vm.nix b/nix/dev-vm.nix deleted file mode 100644 index b7941911..00000000 --- a/nix/dev-vm.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - modulesPath, - lib, - config, - ... -}: { - imports = [ - (modulesPath + "/virtualisation/qemu-vm.nix") - ./solr-module.nix - ./services.nix - ]; - - services.openssh = { - enable = true; - settings.PermitRootLogin = "yes"; - }; - - users.users.root = { - password = "root"; - openssh.authorizedKeys.keyFiles = [ ./dev-vm-key.pub ]; - }; - i18n = {defaultLocale = "de_DE.UTF-8";}; - console.keyMap = "de"; - - networking = { - hostName = "renku-search-testvm"; - }; - - virtualisation.memorySize = 4096; - - virtualisation.forwardPorts = [ - { - from = "host"; - host.port = 10022; - guest.port = 22; - } - { - from = "host"; - host.port = 18983; - guest.port = 8983; - } - { - from = "host"; - host.port = 16379; - guest.port = 6379; - } - { - from = "host"; - host.port = 8088; - guest.port = 80; - } - ]; - documentation.enable = false; -} diff --git a/nix/openapi-doc.nix b/nix/openapi-doc.nix deleted file mode 100644 index c0312830..00000000 --- a/nix/openapi-doc.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ - runCommand, - writeText, - swagger-ui, -}: let - indexhtml = writeText "index.html" '' - - - - - - - SwaggerUI - - - -
- - - - - - ''; -in - runCommand "openapi-docs" {} '' - mkdir $out - ln -snf ${swagger-ui}/lib/node_modules/swagger-ui/dist/* $out/ - ln -snf ${indexhtml} $out/index.html - '' diff --git a/nix/scripts/recreate-container b/nix/scripts/recreate-container deleted file mode 100644 index 7b96c570..00000000 --- a/nix/scripts/recreate-container +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -cnt=''${RS_CONTAINER:-rsdev} -if nixos-container list | grep $cnt > /dev/null; then - echo "Destroying container $cnt" - sudo nixos-container destroy $cnt -fi -echo "Creating and starting container $cnt ..." -sudo nixos-container create $cnt --flake . -sudo nixos-container start $cnt diff --git a/nix/scripts/redis-push b/nix/scripts/redis-push index 0cbaf7c2..79860760 100644 --- a/nix/scripts/redis-push +++ b/nix/scripts/redis-push @@ -7,8 +7,9 @@ fi redis_host=${RS_REDIS_HOST:-rsdev} redis_port=${RS_REDIS_PORT:-6379} +queue_name=$RS_REDIS_QUEUE_projectCreated header='{"source":"dev","type":"project.created","dataContentType":"application/avro+json","schemaVersion":"1","time":0,"requestId":"r1"}' -payload=$(jq --null-input --arg id "$1" --arg name "$2" --arg slug "$1/$2" '{"id":$id,"name":$name,"slug":$slug, "repositories":[],"visibility":"PUBLIC","description":{"string":"my project description"},"createdBy":"dev","creationDate":0,"members":[]}') +payload=$(jq --null-input --arg id "$1" --arg name "$2" --arg slug "$1/$2" '{"id":$id,"name":$name,"slug":$slug, "repositories":[],"visibility":"PUBLIC","description":{"string":"my project description"},"createdBy":"dev","creationDate":0}') -redis-cli -h $redis_host -p $redis_port XADD events '*' headers "$header" payload "$payload" +redis-cli -h $redis_host -p $redis_port XADD $queue_name '*' headers "$header" payload "$payload" diff --git a/nix/services.nix b/nix/services.nix index 467fd8ba..1db48b67 100644 --- a/nix/services.nix +++ b/nix/services.nix @@ -4,28 +4,19 @@ pkgs, ... }: { - services.solr = { + services.dev-solr = { enable = true; + cores = ["rsdev-test" "core-test1" "core-test2" "core-test3" "search-core-test"]; + heap = 512; }; - services.redis.servers.search = { + services.dev-redis = { enable = true; - port = 6379; - bind = "0.0.0.0"; - openFirewall = true; - settings = { - "protected-mode" = "no"; - }; + instance = "search"; }; - services.nginx = { + services.openapi-docs = { enable = true; - virtualHosts.rsdev.locations."/" = { - root = "${pkgs.openapi-doc}"; - }; - }; - - networking = { - firewall.allowedTCPPorts = [8983 80]; + openapi-spec = "http://localhost:8080/search/spec.json"; }; } diff --git a/nix/solr-module.nix b/nix/solr-module.nix deleted file mode 100644 index d5f72798..00000000 --- a/nix/solr-module.nix +++ /dev/null @@ -1,85 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: let - cfg = config.services.solr; -in { - ## interface - options = with lib; { - services.solr = { - enable = mkOption { - default = false; - description = "Whether to enable solr."; - }; - bindAddress = mkOption { - type = types.str; - default = "0.0.0.0"; - description = "The address to bind to"; - }; - port = mkOption { - type = types.int; - default = 8983; - description = "The port solr is listening on."; - }; - heap = mkOption { - type = types.int; - default = 2048; - description = "The heap setting in megabytes"; - }; - home-dir = mkOption { - type = types.str; - default = "/var/solr/data"; - description = "Home dir of solr, to store the data"; - }; - }; - }; - - ## implementation - config = lib.mkIf config.services.solr.enable { - # Create a user for solr - users.users.solr = { - isNormalUser = false; - isSystemUser = true; - group = "solr"; - useDefaultShell = true; - }; - users.groups = {solr = {};}; - - # to allow playing with the solr cli - environment.systemPackages = [pkgs.solr]; - - environment.etc = {solr = {source = "${pkgs.solr}/server/solr";};}; - - # Create directories for storage - systemd.tmpfiles.rules = [ - "d /var/solr 0755 solr solr - -" - "d /var/solr/data 0755 solr solr - -" - "d /var/solr/logs 0755 solr solr - -" - ]; - - systemd.services.solr = { - enable = true; - description = "Apache Solr"; - wantedBy = ["multi-user.target"]; - path = with pkgs; [solr lsof coreutils procps gawk]; - environment = { - SOLR_PORT = toString cfg.port; - SOLR_JETTY_HOST = cfg.bindAddress; - SOLR_HEAP = "${toString cfg.heap}m"; - SOLR_PID_DIR = "/var/solr"; - SOLR_HOME = "${cfg.home-dir}"; - SOLR_LOGS_DIR = "/var/solr/logs"; - }; - serviceConfig = { - ExecStart = "${pkgs.solr}/bin/solr start -f -Dsolr.modules=analysis-extras"; - ExecStop = "${pkgs.solr}/bin/solr stop"; - LimitNOFILE = "65000"; - LimitNPROC = "65000"; - User = "solr"; - Group = "solr"; - }; - }; - }; -} diff --git a/nix/solr.nix b/nix/solr.nix deleted file mode 100644 index 15ddc58e..00000000 --- a/nix/solr.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ - lib, - stdenv, - fetchurl, - makeWrapper, - jre, -}: -# https://discourse.nixos.org/t/solr-has-been-removed-what-are-my-options/33504/3 -stdenv.mkDerivation rec { - pname = "solr"; - version = "9.4.1"; - - src = fetchurl { - url = "mirror://apache/solr/${pname}/${version}/${pname}-${version}.tgz"; - sha256 = "sha256-QQFdemk/76S4pTf10Jgq2ujxPzgu3znJSjSX+bN4MlA="; - }; - - nativeBuildInputs = [makeWrapper]; - - installPhase = '' - mkdir -p $out - cp -r * $out/ - rm -rf $out/bin/init.d - rm $out/bin/postlogs - rm $out/bin/install_solr_service.sh - rm $out/bin/solr.in.sh - rm $out/bin/*.cmd - - wrapProgram $out/bin/solr --set JAVA_HOME "${jre}" - wrapProgram $out/bin/post --set JAVA_HOME "${jre}" - ''; - - meta = with lib; { - homepage = "https://lucene.apache.org/solr/"; - description = "Open source enterprise search platform from the Apache Lucene project"; - license = licenses.asl20; - latforms = platforms.all; - maintainers = with maintainers; []; - }; -} diff --git a/nix/swagger-ui.nix b/nix/swagger-ui.nix deleted file mode 100644 index b0e2e81e..00000000 --- a/nix/swagger-ui.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ - buildNpmPackage, - nodejs, - cypress, - fetchFromGitHub, -}: -buildNpmPackage { - name = "swagger-ui"; - - buildInputs = [nodejs cypress]; - - src = fetchFromGitHub { - owner = "swagger-api"; - repo = "swagger-ui"; - rev = "v5.11.7"; - sha256 = "sha256-p/KCG+jf0J+KbZpBHwKm4ypt+APpq9L8jzseG0YssSg="; - }; - - npmDepsHash = "sha256-I6InBOAil6F4zFz7YsysDsrqTmhUzkSuEOy9fyBmisA="; - - #https://gist.github.com/r-k-b/2485f977b476aa3f76a47329ce7f9ad4 - CYPRESS_INSTALL_BINARY = "0"; - CYPRESS_RUN_BINARY = "${cypress}/bin/Cypress"; - - npmBuildScript = "build:standalone"; -}