appid: Remove special check for code hosting domains

git subprocess seems to randomly fail on Flathub commit stage

Use `check_url` for all except `io.gitlab`. io.gitlab must return
HTTP 200 OK as it returns 302 on non existent pages like
https://ghost.gitlab.io

For gitlab hostings on `gitlab.gnome.org` or `gitlab.freedesktop.org`,
just return the base domain

Try https domain only for known code hosting sites

flatpak_builder_lint/checks/appid.py

@@ -58,65 +58,29 @@ def _validate(self, appid: Optional[str], is_extension: bool) -> None:
                 return
             if domainutils.is_app_on_flathub(appid):
                 return
-            if appid.startswith(
-                (
-                    "io.github.",
-                    "io.gitlab.",
-                    "io.frama.",
-                    "page.codeberg.",
-                    "io.sourceforge.",
-                    "net.sourceforge.",
-                    "org.gnome.gitlab.",
-                    "org.freedesktop.gitlab.",
+            appid_domain = domainutils.get_domain(appid)
+            if appid_domain is None:
+                self.errors.add("appid-domain-not-found")
+                self.info.add(
+                    f"appid-domain-not-found: Domain for {appid}"
+                    + " cannot be determined"
                 )
-            ):
-                appid_code_host = domainutils.get_code_hosting_url(appid)
-                if appid_code_host is None:
-                    self.errors.add("appid-code-host-not-found")
-                    self.info.add(
-                        f"appid-code-host-not-found: Code hosting url for {appid}"
-                        + " cannot be determined"
-                    )
-                    return
-                else:
-                    if isinstance(appid_code_host, list):
-                        if not (
-                            domainutils.check_git(appid_code_host[0])
-                            or domainutils.check_git(appid_code_host[1])
-                        ):
-                            self.warnings.add("appid-code-host-not-reachable")
-                            self.info.add(
-                                f"appid-code-host-not-reachable: {appid_code_host} not reachable"
-                            )
-                    else:
-                        if appid_code_host.startswith(
-                            "https://sourceforge.net/projects/"
-                        ):
-                            if not domainutils.check_url(appid_code_host):
-                                self.warnings.add("appid-code-host-not-reachable")
-                                self.info.add(
-                                    f"appid-code-host-not-reachable: {appid_code_host}"
-                                    + " not reachable"
-                                )
-                        else:
-                            if not domainutils.check_git(appid_code_host):
-                                self.warnings.add("appid-code-host-not-reachable")
-                                self.info.add(
-                                    f"appid-code-host-not-reachable: {appid_code_host}"
-                                    + " not reachable"
-                                )
+                return
             else:
-                appid_domain = domainutils.get_domain(appid)
-                if appid_domain is None:
-                    self.errors.add("appid-domain-not-found")
-                    self.info.add(
-                        f"appid-domain-not-found: Domain for {appid}"
-                        + " cannot be determined"
-                    )
-                    return
+                url_http = f"http://{appid_domain}"
+                url_https = f"https://{appid_domain}"
+                if appid_domain.endswith(
+                    (".github.io", ".gitlab.io", ".codeberg.io", ".frama.io")
+                ) or appid_domain.startswith("sourceforge.net/projects/"):
+                    if appid_domain.endswith(".gitlab.io"):
+                        if not domainutils.check_url_ok(url_https):
+                            self.errors.add("appid-url-not-reachable")
+                            self.info.add(f"appid-url-not-reachable: Tried {url_https}")
+                    else:
+                        if not domainutils.check_url(url_https):
+                            self.errors.add("appid-url-not-reachable")
+                            self.info.add(f"appid-url-not-reachable: Tried {url_https}")
                 else:
-                    url_http = f"http://{appid_domain}"
-                    url_https = f"https://{appid_domain}"
                     if not (
                         domainutils.check_url(url_https)
                         or domainutils.check_url(url_http)

flatpak_builder_lint/domainutils.py

@@ -1,6 +1,3 @@
-import subprocess
-from typing import List
-
 import requests
 
 
@@ -16,20 +13,16 @@ def check_url(url: str) -> bool:
     return ret
 
 
-def check_git(url: str) -> bool:
-    success = False
+def check_url_ok(url: str) -> bool:
+    assert url.endswith(".gitlab.io")
+    ret = False
     try:
-        ret = subprocess.run(
-            ["git", "ls-remote", "-q", "--exit-code", url, "HEAD"],
-            timeout=10,
-            capture_output=True,
-        )
-        if ret.returncode == 0:
-            success = True
-    except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as err:
-        print(err)
-
-    return success
+        r = requests.get(url, allow_redirects=False, timeout=10)
+        if r.status_code == 200:
+            ret = True
+    except requests.exceptions.RequestException:
+        pass
+    return ret
 
 
 def demangle(name: str) -> str:
@@ -40,27 +33,21 @@ def demangle(name: str) -> str:
 
 
 def get_domain(appid: str) -> str | None:
-    assert not appid.startswith(
-        (
-            "io.github.",
-            "io.gitlab.",
-            "io.frama.",
-            "page.codeberg.",
-            "io.sourceforge.",
-            "net.sourceforge.",
-            "org.gnome.gitlab.",
-            "org.freedesktop.gitlab.",
-        )
-    )
     domain = None
-    if appid.startswith("org.gnome.") and not appid.startswith("org.gnome.gitlab."):
+    if appid.startswith("org.gnome."):
         domain = "gnome.org"
     elif appid.startswith("org.kde."):
         domain = "kde.org"
-    elif appid.startswith("org.freedesktop.") and not appid.startswith(
-        "org.freedesktop.gitlab."
-    ):
+    elif appid.startswith("org.freedesktop."):
         domain = "freedesktop.org"
+    elif appid.startswith(("io.github.", "io.gitlab.", "page.codeberg.", "io.frama.")):
+        tld = appid.split(".")[0]
+        demangled = [demangle(i) for i in appid.split(".")[1:3]]
+        demangled.insert(0, tld)
+        domain = ".".join(reversed(demangled)).lower()
+    elif appid.startswith(("io.sourceforge.", "net.sourceforge.")):
+        proj = demangle(appid.split(".")[2])
+        domain = f"sourceforge.net/projects/{proj}".lower()
     else:
         tld = appid.split(".")[0]
         demangled = [demangle(i) for i in appid.split(".")[:-1][1:]]
@@ -70,73 +57,5 @@ def get_domain(appid: str) -> str | None:
     return domain
 
 
-def get_code_hosting_url(appid: str) -> str | List[str] | None:
-    assert appid.startswith(
-        (
-            "io.github.",
-            "io.gitlab.",
-            "io.frama.",
-            "page.codeberg.",
-            "io.sourceforge.",
-            "net.sourceforge.",
-            "org.gnome.gitlab.",
-            "org.freedesktop.gitlab.",
-        )
-    )
-    code_host: str | None | List[str] = None
-    if appid.startswith(("io.sourceforge.", "net.sourceforge.")):
-        sf_proj = appid.split(".")[2:3][0]
-        code_host = f"https://sourceforge.net/projects/{sf_proj}".lower()
-    if appid.startswith("org.gnome.gitlab."):
-        [user, proj] = appid.split(".")[3:5]
-        user = demangle(user)
-        code_host = f"https://gitlab.gnome.org/{user}/{proj}.git".lower()
-    if appid.startswith("org.freedesktop.gitlab."):
-        [user, proj] = appid.split(".")[3:5]
-        user = demangle(user)
-        code_host = f"https://gitlab.freedesktop.org/{user}/{proj}.git".lower()
-    if len(appid.split(".")) == 4:
-        [sld, user, proj] = appid.split(".")[1:4]
-        sld = demangle(sld)
-        user = demangle(user)
-
-        if appid.startswith("io.github."):
-            code_host = f"https://github.com/{user}/{proj}.git".lower()
-        if appid.startswith("io.gitlab."):
-            code_host = f"https://gitlab.com/{user}/{proj}.git".lower()
-        if appid.startswith("io.frama."):
-            code_host = f"https://framagit.org/{user}/{proj}.git".lower()
-        if appid.startswith("page.codeberg."):
-            code_host = f"https://codeberg.org/{user}/{proj}.git".lower()
-
-    if len(appid.split(".")) == 5:
-        [sld, user, proj1, proj2] = appid.split(".")[1:5]
-        sld = demangle(sld)
-        user = demangle(user)
-        proj1 = demangle(proj1)
-
-        if appid.startswith("io.github."):
-            code_host = [
-                f"https://github.com/{user}/{proj1}.git".lower(),
-                f"https://github.com/{user}/{proj2}.git".lower(),
-            ]
-        if appid.startswith("io.gitlab."):
-            code_host = [
-                f"https://gitlab.com/{user}/{proj1}.git".lower(),
-                f"https://gitlab.com/{user}/{proj2}.git".lower(),
-            ]
-        if appid.startswith("io.frama."):
-            code_host = [
-                f"https://framagit.org/{user}/{proj1}.git".lower(),
-                f"https://framagit.org/{user}/{proj2}.git".lower(),
-            ]
-        if appid.startswith("page.codeberg."):
-            code_host = [
-                f"https://codeberg.org/{user}/{proj1}.git".lower(),
-                f"https://codeberg.org/{user}/{proj2}.git".lower(),
-            ]
-    return code_host
-
-
 def is_app_on_flathub(appid: str) -> bool:
     return check_url(f"https://flathub.org/api/v2/summary/{appid}")

tests/builddir/wrong-rdns-appid/metadata

@@ -1,5 +1,5 @@
 [Application]
-name=org.freedesktop.gitlab.foo.bar
+name=io.gitlab.foo.bar
 runtime=org.gnome.Platform/x86_64/45
 sdk=org.gnome.Sdk/x86_64/45
 command=foo

tests/manifests/domain_checks/io.github.flathub.flathub.json → tests/manifests/domain_checks/io.github.flatpak.flatpak.json

@@ -1,5 +1,5 @@
 {
-    "app-id": "io.github.flathub.flathub",
+    "app-id": "io.github.flatpak.flatpak",
     "runtime": "foo",
     "sdk": "bar",
     "command": "foo",

tests/manifests/domain_checks/org.freedesktop.gitlab.foo.bar.json → tests/manifests/domain_checks/io.gitlab.foo.bar.json

@@ -1,5 +1,5 @@
 {
-    "app-id": "org.freedesktop.gitlab.foo.bar",
+    "app-id": "io.gitlab.foo.bar",
     "runtime": "foo",
     "sdk": "bar",
     "command": "foo",

tests/test_builddir.py

@@ -15,10 +15,10 @@ def test_builddir_appid() -> None:
     assert "appstream-metainfo-missing" in found_errors
 
 
-def test_builddir_appid_code_host_not_reachable() -> None:
+def test_builddir_url_not_reachable() -> None:
     ret = run_checks("tests/builddir/wrong-rdns-appid")
-    found_warnings = set(ret["warnings"])
-    assert "appid-code-host-not-reachable" in found_warnings
+    found_errors = set(ret["errors"])
+    assert "appid-url-not-reachable" in found_errors
 
 
 def test_builddir_finish_args() -> None:
@@ -273,7 +273,6 @@ def test_min_success_metadata() -> None:
     not_founds = {
         "appid-too-many-components-for-app",
         "metainfo-missing-launchable-tag",
-        "appid-code-host-not-found",
         "appid-domain-not-found",
         "appid-url-not-reachable",
     }

tests/test_manifest.py

@@ -27,33 +27,28 @@ def test_appid_too_many_cpts() -> None:
     assert {"appid-too-many-components-for-app"} == errors
 
 
-def test_appid_code_host_not_reachable() -> None:
+def test_appid_url_not_reachable() -> None:
     for i in (
         "tests/manifests/domain_checks/io.github.ghost.bar.json",
         "tests/manifests/domain_checks/io.github.ghost.foo.bar.json",
-        "tests/manifests/domain_checks/org.freedesktop.gitlab.foo.bar.json",
+        "tests/manifests/domain_checks/io.gitlab.foo.bar.json",
         "tests/manifests/domain_checks/io.sourceforge.wwwwwwwwwwwwwwww.bar.json",
+        "tests/manifests/domain_checks/ch.wwwwww.bar.json",
     ):
         ret = run_checks(i)
-        warnings = set(ret["warnings"])
-        assert "appid-code-host-not-reachable" in warnings
+        errors = set(ret["errors"])
+        assert "appid-url-not-reachable" in errors
 
 
-def test_appid_code_host_is_reachable() -> None:
+def test_appid_url_is_reachable() -> None:
     for i in (
-        "tests/manifests/domain_checks/io.github.flathub.flathub.json",
+        "tests/manifests/domain_checks/io.github.flatpak.flatpak.json",
         "tests/manifests/domain_checks/org.gnome.gitlab.YaLTeR.Identity.json",
     ):
         ret = run_checks(i)
         assert "errors" not in ret
 
 
-def test_appid_url_not_reachable() -> None:
-    ret = run_checks("tests/manifests/domain_checks/ch.wwwwww.bar.json")
-    errors = set(ret["errors"])
-    assert {"appid-url-not-reachable"} == errors
-
-
 def test_appid_on_flathub() -> None:
     # encom.eu.org does not exist
     ret = run_checks("tests/manifests/domain_checks/org.eu.encom.spectral.json")