No hook but cache

README.md

@@ -116,16 +116,6 @@ the dependencies:
     export CONTAINER_HOST=unix:///run/user/1001/podman/podman.sock
     poetry run rq worker
 
-#### Update targets
-
-To update the list of available targets, run:
-
-    poetry run python3 misc/update_all_targets.py
-
-This may be added to a cron job to update the targets regularly. The script must
-be changed in case you want to update the targets from a different source or run
-the server on a different port.
-
 ### API
 
 The API is documented via _OpenAPI_ and can be viewed interactively on the

asu/build.py

@@ -21,8 +21,8 @@
     get_packages_hash,
     get_podman,
     get_request_hash,
-    parse_manifest,
     is_snapshot_build,
+    parse_manifest,
     report_error,
     run_cmd,
 )
@@ -78,6 +78,7 @@ def build(build_request: BuildRequest, job=None):
         if settings.squid_cache:
             environment.update(
                 {
+                    "UPSTREAM_URL": settings.upstream_url.replace("https", "http"),
                     "use_proxy": "on",
                     "http_proxy": "http://127.0.0.1:3128",
                 }

asu/build_request.py

@@ -1,4 +1,5 @@
 from typing import Annotated
+
 from pydantic import BaseModel, Field
 
 from asu.config import settings

asu/config.py

@@ -44,6 +44,7 @@ def release(branch_off_rev, enabled=True):
     return {
         "path": "releases/{version}",
         "enabled": enabled,
+        "snapshot": False,
         "path_packages": "DEPRECATED",
         "branch_off_rev": branch_off_rev,
         "package_changes": package_changes(branch_off_rev),
@@ -64,13 +65,13 @@ class Settings(BaseSettings):
     max_defaults_length: int = 20480
     repository_allow_list: list = []
     base_container: str = "ghcr.io/openwrt/imagebuilder"
-    update_token: Union[str, None] = "foobar"
     container_host: str = "localhost"
     container_identity: str = ""
     branches: dict = {
         "SNAPSHOT": {
             "path": "snapshots",
             "enabled": True,
+            "snapshot": True,
             "path_packages": "DEPRECATED",
             "package_changes": package_changes(),
         },
@@ -79,7 +80,7 @@ class Settings(BaseSettings):
         "22.03": release(19160),
         "21.02": release(15812, enabled=True),  # Enabled for now...
     }
-    server_stats: str = "/stats"
+    server_stats: str = ""
     log_level: str = "INFO"
     squid_cache: bool = False
 

asu/main.py

@@ -1,42 +1,33 @@
 import logging
-from contextlib import asynccontextmanager
+from collections import defaultdict
+from datetime import datetime
 from pathlib import Path
+from typing import Union
 
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-from fastapi_cache import FastAPICache
-from fastapi_cache.backends.inmemory import InMemoryBackend
-from fastapi_cache.coder import PickleCoder
-from fastapi_cache.decorator import cache
 
 from asu import __version__
 from asu.config import settings
 from asu.routers import api
 from asu.util import (
-    get_redis_client,
+    client_get,
+    get_branch,
+    is_post_kmod_split_build,
     parse_feeds_conf,
-    parse_packages_file,
     parse_kernel_version,
-    is_post_kmod_split_build,
+    parse_packages_file,
+    reload_targets,
+    reload_versions,
 )
 
 logging.basicConfig(encoding="utf-8", level=settings.log_level)
 
 base_path = Path(__file__).resolve().parent
 
-
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    logging.info("ASU server starting up")
-    FastAPICache.init(InMemoryBackend())
-    yield
-    # Any shutdown tasks here...
-    logging.info("ASU server shutting down")
-
-
-app = FastAPI(lifespan=lifespan)
+app = FastAPI()
 app.include_router(api.router, prefix="/api/v1")
 
 (settings.public_path / "json").mkdir(parents=True, exist_ok=True)
@@ -47,39 +38,31 @@ async def lifespan(app: FastAPI):
 
 templates = Jinja2Templates(directory=base_path / "templates")
 
+app.versions = []
+reload_versions(app)
+logging.info(f"Found {len(app.versions)} versions")
+
+app.targets = defaultdict(list)
+app.profiles = defaultdict(lambda: defaultdict(dict))
+
 
 @app.get("/", response_class=HTMLResponse)
-@cache(expire=600, coder=PickleCoder)
 def index(request: Request):
-    redis_client = get_redis_client()
-
-    branches = dict(
-        map(
-            lambda b: (
-                b,
-                {
-                    "versions": sorted(redis_client.smembers(f"versions:{b}")),
-                },
-            ),
-            sorted(redis_client.smembers("branches")),
-        )
-    )
-
     return templates.TemplateResponse(
         request=request,
         name="overview.html",
         context=dict(
-            branches=branches,
+            versions=app.versions,
             defaults=settings.allow_defaults,
             version=__version__,
             server_stats=settings.server_stats,
+            max_custom_rootfs_size_mb=settings.max_custom_rootfs_size_mb,
         ),
     )
 
 
 @app.get("/json/v1/{path:path}/index.json")
-@cache(expire=600)
-def json_v1_target_index(path: str) -> dict[str, str]:
+def json_v1_target_index(path: str) -> dict[str, Union[str, dict[str, str]]]:
     base_path: str = f"{settings.upstream_url}/{path}"
     base_packages: dict[str, str] = parse_packages_file(f"{base_path}/packages")
     if is_post_kmod_split_build(path):
@@ -93,7 +76,6 @@ def json_v1_target_index(path: str) -> dict[str, str]:
 
 
 @app.get("/json/v1/{path:path}/{arch:path}-index.json")
-@cache(expire=600)
 def json_v1_arch_index(path: str, arch: str):
     feed_url: str = f"{settings.upstream_url}/{path}/{arch}"
     feeds: list[str] = parse_feeds_conf(feed_url)
@@ -103,6 +85,88 @@ def json_v1_arch_index(path: str, arch: str):
     return packages
 
 
+@app.get("/json/v1/{path:path}/targets/{target:path}/{profile:path}.json")
+def json_v1_profile(path: str, target: str, profile: str):
+    metadata: dict = client_get(
+        f"{settings.upstream_url}/{path}/targets/{target}/profiles.json"
+    ).json()
+    profiles: dict = metadata.pop("profiles", {})
+    if profile not in profiles:
+        return {}
+
+    return {
+        **metadata,
+        **profiles[profile],
+        "id": profile,
+        "build_at": datetime.utcfromtimestamp(
+            int(metadata.get("source_date_epoch", 0))
+        ).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+    }
+
+
+def generate_latest():
+    response = client_get(f"{settings.upstream_url}/.versions.json")
+
+    versions_upstream = response.json()
+    latest = [
+        versions_upstream["stable_version"],
+        versions_upstream["oldstable_version"],
+    ]
+
+    if versions_upstream["upcoming_version"]:
+        latest.insert(0, versions_upstream["upcoming_version"])
+    return latest
+
+
+@app.get("/json/v1/latest.json")
+def json_v1_latest():
+    latest = generate_latest()
+    return {"latest": latest}
+
+
+def generate_branches():
+    branches = dict(**settings.branches)
+
+    for branch in branches:
+        branches[branch]["versions"] = []
+        branches[branch]["name"] = branch
+
+    for version in app.versions:
+        branch_name = get_branch(version)["name"]
+        branches[branch_name]["versions"].append(version)
+
+    for branch in branches:
+        version = branches[branch]["versions"][0]
+        if not app.targets[version]:
+            reload_targets(app, version)
+
+        branches[branch]["targets"] = app.targets[version]
+
+    return branches
+
+
+@app.get("/json/v1/branches.json")
+def json_v1_branches():
+    return list(generate_branches().values())
+
+
+@app.get("/json/v1/overview.json")
+def json_v1_overview():
+    overview = {
+        "latest": generate_latest(),
+        "branches": generate_branches(),
+        "upstream_url": settings.upstream_url,
+        "server": {
+            "version": __version__,
+            "contact": "[email protected]",
+            "allow_defaults": settings.allow_defaults,
+            "repository_allow_list": settings.repository_allow_list,
+        },
+    }
+
+    return overview
+
+
 app.mount(
     "/json",
     StaticFiles(directory=settings.public_path / "json"),