feat: migrate from janky custom manifest format to yaml

README.md

@@ -59,17 +59,19 @@ More info on using neovim with nix can be found here: [NixOS Neovim](https://nix
 
 ### How to add a new plugin
 
-#### 1. Add the plugin to manifest.txt:
+#### 1. Add the plugin to manifest.yaml:
+
+The new yaml format has the following format
 
 ```
-# Examples
-
-haringsrob/nvim_context_vt
-sourcehut:henriquehbr/ataraxis.lua
-gitlab:yorickpeterse/nvim-pqf
-williamboman/mason.nvim:45b9a4da776d9fb017960b3ac7241161fb7bc578 
-foo/bar::baz                   --> renamed to baz
-foo/bar:dev                    --> using dev branch
+- owner: nvim-telescope
+  repo: telescope.nvim
+  # the following keys are optional
+  branch: ... # explicitly select branch
+  custom_name: ... # set custom name, used to fix name clashes
+  license: ... # specify license
+  commit: ... # specify commit to use, can be used when newest version is broken
+  warning: ... # add a waning that will be displayed when using the plugin, will also be visible in plugins.md
 ```
 
 Supported are Github (default), SourceHut, and GitLab.

bin/pyproject.toml

@@ -14,6 +14,7 @@ requests = "^2.28.2"
 cleo = "^2.0.1"
 jsonpickle = "*"
 dateparser = "1.1.8"
+PyYAML = "^6.0"
 
 [tool.poetry.group.test.dependencies]
 pytest-cov = "^4.0.0"

bin/update_vim_plugins/__main__.py

@@ -3,13 +3,15 @@
 from .update import UpdateCommand
 from .cleanup import CleanUpCommand
 from .fetch import FetchCommand
+from .regenerate import RegenerateCommand
 
 
 def main():
     application = Application()
     application.add(UpdateCommand())
     application.add(CleanUpCommand())
     application.add(FetchCommand())
+    application.add(RegenerateCommand())
     application.run()
 
 

bin/update_vim_plugins/cleanup.py

@@ -11,14 +11,15 @@ def handle(self):
         """Main command function"""
 
         # all cleaning up will be done during reading and writing automatically
-        manifest = read_manifest_to_spec()
+        #  manifest = read_manifest_to_spec()
+        manifest = read_manifest_yaml_to_spec()
         blacklist = read_blacklist_to_spec()
 
         new_manifest = [ spec for spec in manifest if spec not in blacklist ]
 
         new_manifest_filterd = self.filter_renamed(new_manifest)
 
-        write_manifest_from_spec(new_manifest_filterd)
+        write_manifest_yaml_from_spec(new_manifest_filterd)
 
         self.line("<comment>Done</comment>")
 

bin/update_vim_plugins/fetch.py

@@ -13,27 +13,27 @@ class FetchCommand(Command):
     def handle(self):
         """Main command function"""
 
-        specs = read_manifest()
+        specs = read_manifest_yaml_to_spec()
         specs += self.fetch_awesome()
         specs += self.fetch_m15a()
 
-        write_manifest(specs)
+        write_manifest_yaml_from_spec(specs)
 
         self.line("<comment>Done</comment>")
 
-    def fetch_m15a(self) -> list[str]:
+    def fetch_m15a(self) -> list[PluginSpec]:
         self.line(f"<info>Fetching from m15a's repo</info>")
 
         manifest = urlopen(M15A_MANIFEST).read()
         manifest = str(manifest, 'utf-8')
         manifest = manifest.split("\n")
 
         specs = list(filter(lambda x: x != "", manifest))
-        specs = [ p for p in specs ]
+        specs = [ PluginSpec.from_spec(p) for p in specs ]
 
         return specs
 
-    def fetch_awesome(self) -> list[str]:
+    def fetch_awesome(self) -> list[PluginSpec]:
         self.line(f"<info>Fetching from awesome-neovim</info>")
 
         readme = urlopen(AWESOME_NEOVIM_README).read()
@@ -108,6 +108,8 @@ def fetch_awesome(self) -> list[str]:
 
             specs.append(spec)
 
+        specs = [ PluginSpec.from_spec(s) for s in specs ]
+
         return specs
 
 

bin/update_vim_plugins/helpers.py

@@ -1,25 +1,34 @@
 from .spec import PluginSpec
+import yaml
+from pprint import pprint
+import subprocess
 
-MANIFEST_FILE = "./manifest.txt"
+from .plugin import VimPlugin
+
+#  MANIFEST_FILE = "./manifest.txt"
+MANIFEST_YAML = "./manifest.yaml"
 BLACKLIST_FILE = "./blacklist.txt"
 PKGS_FILE = "./pkgs/vim-plugins.nix"
 AWESOME_NEOVIM_README = "https://raw.githubusercontent.com/rockerBOO/awesome-neovim/main/README.md"
 M15A_MANIFEST = "https://raw.githubusercontent.com/m15a/nixpkgs-vim-extra-plugins/main/manifest.txt"
 JSON_FILE = "./.plugins.json"
 PLUGINS_LIST_FILE = "./plugins.md"
 
-def read_manifest() -> list[str]:
-    with open(MANIFEST_FILE, "r") as file:
-        specs = set([ spec.strip() for spec in file.readlines() ])
+#  def read_manifest_to_spec() -> list[PluginSpec]:
+    #  manifest = read_manifest()
+    #  specs = [ PluginSpec.from_spec(spec.strip()) for spec in manifest ]
 
-    return sorted(specs)
+    #  return sorted(specs)
 
-def read_manifest_to_spec() -> list[PluginSpec]:
-    manifest = read_manifest()
-    specs = [ PluginSpec.from_spec(spec.strip()) for spec in manifest ]
+def read_manifest_yaml_to_spec() -> list[PluginSpec]:
+    with open(MANIFEST_YAML, "r") as file:
+        data = yaml.safe_load(file)
+
+    specs = [ PluginSpec.from_yaml(p) for p in data ]
 
     return sorted(specs)
 
+
 def read_blacklist() -> list[str]:
     with open(BLACKLIST_FILE, "r") as file:
         blacklisted_specs = set([ spec.strip() for spec in file.readlines() ])
@@ -32,19 +41,36 @@ def read_blacklist_to_spec() -> list[PluginSpec]:
 
     return sorted(specs)
 
-def write_manifest(specs: list[str]|set[str]):
-    """write specs to manifest file. Does some cleaning up"""
+def write_plugins_nix(plugins: list[VimPlugin]):
+    plugins.sort()
+
+    header = "{ lib, buildVimPlugin, fetchurl, fetchgit }: {"
+    footer = "}"
 
-    with open(MANIFEST_FILE, "w") as file:
+    with open(PKGS_FILE, "w") as file:
+        file.write(header)
+        for plugin in plugins:
+            try:
+                file.write(f"{plugin.to_nix()}\n")
+            except Exception as e:
+                pass
 
-        specs = sorted(set(specs), key=lambda x: x.lower())
-        specs = [ p for p in specs ]
+        file.write(footer)
 
-        for s in specs:
-            file.write(f"{s}\n")
+def format_nix_output():
+    subprocess.run(
+        ["alejandra", PKGS_FILE],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
 
-def write_manifest_from_spec(specs: list[PluginSpec]):
+
+def write_manifest_yaml_from_spec(specs: list[PluginSpec]):
     """write specs to manifest file. Does some cleaning up"""
 
-    strings = [ f"{spec}" for spec in specs ]
-    write_manifest(strings)
+    strings = [ p.to_dict() for p in specs ]
+    y = yaml.dump(strings, default_flow_style=False, sort_keys=False)
+    with open(MANIFEST_YAML, "w") as file:
+        file.write(y)
+        file.write("\n")
+

bin/update_vim_plugins/plugin.py

@@ -26,24 +26,47 @@ class VimPlugin:
     description: str = "No description"
     homepage: str
     license: License
-    source_line: str
     checked: date = datetime.now().date()
+    warning: str | None = None
+
+    @property
+    def id(self) -> str:
+        if not hasattr(self, 'repo'): # WARN: should be removed after a few runs, only needed to handle old .plugin.json entries
+            self.repo = self.source_line.split("/")[1]
+
+        return f"{self.owner}/{self.repo}"
 
     def to_nix(self):
         """Return the nix expression for this plugin."""
         meta = f'with lib; {{ description = "{self.description}"; homepage = "{self.homepage}"; license = with licenses; [ {self.license.value} ]; }}'
-        return f'/* Generated from: {self.source_line} */ {self.name} = buildVimPlugin {{ pname = "{self.name}";  version = "{self.version}"; src = {self.source.get_nix_expression()}; meta = {meta}; }};'
+
+        if self.warning:
+            warning = f"lib.warn \"Warning for '{self.name}': {self.warning}\""
+        else:
+            warning = ""
+
+        return f'''
+            /* Generated from: {self.id} */
+            {self.name} = {warning} buildVimPlugin {{
+                pname = "{self.name}";
+                version = "{self.version}";
+                src = {self.source.get_nix_expression()};
+                meta = {meta};
+            }};
+        '''
 
     def to_json(self):
         """Serizalize the plugin to json"""
         return jsonpickle.encode(self)
 
     def to_markdown(self):
-        link = f"[{self.source_line}]({self.homepage})"
+        link = f"[{self.id}]({self.homepage})"
         version = f"{self.version}"
         package_name = f"{self.name}"
 
-        return f"| {link} | {version} | `{package_name}` |"
+        warning = f"{self.warning or ''}"
+
+        return f"| {link} | {version} | `{package_name}` | {warning}"
 
     def __lt__(self, o: object) -> bool:
         if not isinstance(o, VimPlugin):
@@ -67,21 +90,24 @@ class GitHubPlugin(VimPlugin):
     def __init__(self, plugin_spec: PluginSpec) -> None:
         """Initialize a GitHubPlugin."""
 
-        full_name = f"{plugin_spec.owner}/{plugin_spec.repo}"
+        owner = plugin_spec.owner
+        repo = plugin_spec.repo
+        full_name = f"{owner}/{repo}"
         repo_info = self._api_call(f"repos/{full_name}")
         default_branch = plugin_spec.branch or repo_info["default_branch"]
         api_callback = self._api_call(f"repos/{full_name}/commits/{default_branch}")
         latest_commit = api_callback["commit"]
         sha = api_callback["sha"]
 
         self.name = plugin_spec.name
-        self.owner = plugin_spec.owner
+        self.repo = repo
+        self.owner = owner
         self.version = parse(latest_commit["committer"]["date"]).date()
         self.source = UrlSource(f"https://github.com/{full_name}/archive/{sha}.tar.gz")
         self.description = (repo_info.get("description") or "").replace('"', '\\"')
         self.homepage = repo_info["html_url"]
         self.license = plugin_spec.license or License.from_spdx_id((repo_info.get("license") or {}).get("spdx_id"))
-        self.source_line = plugin_spec.line
+        self.warning = plugin_spec.warning
 
     def _api_call(self, path: str, token: str | None = _get_github_token()):
         """Call the GitHub API."""
@@ -107,13 +133,14 @@ def __init__(self, plugin_spec: PluginSpec) -> None:
         sha = latest_commit["id"]
 
         self.name = plugin_spec.name
+        self.repo = plugin_spec.repo
         self.owner = plugin_spec.owner
         self.version = parse(latest_commit["created_at"]).date()
         self.source = UrlSource(f"https://gitlab.com/api/v4/projects/{full_name}/repository/archive.tar.gz?sha={sha}")
         self.description = (repo_info.get("description") or "").replace('"', '\\"')
         self.homepage = repo_info["web_url"]
         self.license = plugin_spec.license or License.from_spdx_id(repo_info.get("license", {}).get("key"))
-        self.source_line = plugin_spec.line
+        self.warning = plugin_spec.warning
 
     def _api_call(self, path: str) -> dict:
         """Call the Gitlab API."""
@@ -144,13 +171,13 @@ def __init__(self, plugin_spec: PluginSpec) -> None:
         sha = latest_commit["id"]
 
         self.name = plugin_spec.name
+        self.repo = plugin_spec.repo
         self.owner = plugin_spec.owner
         self.version = parse(latest_commit["timestamp"]).date()
         self.description = (repo_info.get("description") or "").replace('"', '\\"')
         self.homepage = f"https://git.sr.ht/~{plugin_spec.owner}/{plugin_spec.repo}"
         self.source = GitSource(self.homepage, sha)
         self.license = plugin_spec.license or License.UNKNOWN  # cannot be determined via API
-        self.source_line = plugin_spec.line
 
     def _api_call(self, path: str, token: str | None = _get_sourcehut_token()):
         """Call the SourceHut API."""

bin/update_vim_plugins/regenerate.py

@@ -0,0 +1,31 @@
+from cleo.commands.command import Command
+from cleo.helpers import argument
+import json
+import jsonpickle
+from pprint import pprint
+
+from .helpers import *
+
+class RegenerateCommand(Command):
+    name = "regenerate"
+    description = "Regenerate plugins from .plugins.json"
+
+    def handle(self):
+        """Main command function"""
+
+
+        json_data = None
+        with open(JSON_FILE, "r+") as json_file:
+            json_data = json.load(json_file)
+
+        plugins = [ jsonpickle.decode(plugin_json) for plugin_json in json_data.values() ]
+
+        self.line(f"<info>Generating nix output</info>")
+
+        write_plugins_nix(plugins)
+
+        self.line(f"<info>Formatting nix output</info>")
+
+        format_nix_output()
+
+        self.line("<comment>Done</comment>")