chore(workflows/build): use static robots.txt

mdn · Jan 21, 2025 · c6b91ab · c6b91ab
1 parent d10d71b
commit c6b91ab
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 3 deletions.
diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml
@@ -301,7 +301,8 @@ jobs:
           yarn build:sw
           yarn build:client
           yarn build:ssr
-          yarn tool:legacy build-robots-txt
+
+          cp assets/prod/robots.txt client/build/robots.txt
 
           yarn rari content sync-translated-content
           yarn rari git-history

diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml
@@ -317,7 +317,8 @@ jobs:
           yarn build:sw
           yarn build:client
           yarn build:ssr
-          yarn tool:legacy build-robots-txt
+
+          cp assets/nonprod/robots.txt client/build/robots.txt
 
           yarn rari content sync-translated-content
           yarn rari git-history

diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml
@@ -249,7 +249,8 @@ jobs:
           yarn build:sw
           yarn build:client
           yarn build:ssr
-          yarn tool:legacy build-robots-txt
+
+          cp assets/nonprod/robots.txt client/build/robots.txt
 
           yarn rari content sync-translated-content
           yarn rari git-history

diff --git a/assets/nonprod/robots.txt b/assets/nonprod/robots.txt
@@ -0,0 +1,3 @@
+User-Agent: *
+
+Disallow: /
diff --git a/assets/prod/robots.txt b/assets/prod/robots.txt
@@ -0,0 +1,6 @@
+User-agent: *
+Sitemap: https://developer.mozilla.org/sitemap.xml
+
+Disallow: /api/
+Disallow: /*/files/
+Disallow: /media