feature: support collection of whitelisted hosts in the HTTP header C…

…ontent-Security-Policy (CSP)

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * fix: WEC failed to extract links from some websites, error: Cannot read properties of undefined (reading 'startsWith')
 * dependencies: overwrite the dependency of `puppeteer-har` to address `npm audit` warnings
+* feature: support collection of whitelisted hosts in the HTTP header Content-Security-Policy (CSP)
 
 ## 2.1.2 / 2024-04-25
 

assets/template.pug

@@ -240,6 +240,24 @@ html(xmlns='http://www.w3.org/1999/xhtml', xml:lang='en', lang='en')
             td Low
             td.notrunc= vulnerabilitiesBySeverity['LOW'] ? vulnerabilitiesBySeverity['LOW'].length : 0
 
+
+    h2(id="sec:use-of-csp") Use of Content Security Policies (CSPs)
+
+    p Upon browser request of a web page, websites can indicate in the #[a(href="https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP") Content Security Policy] (CSP) meta data sent along with the requested web page a whitelist of mechanisms, domains and subdomains that browsers must respect when embedding components, such as styles, fonts, beacons, videos, maps, etc.
+
+    if hosts.contentSecurityPolicy.firstParty.length > 0
+      ol
+        each host in hosts.contentSecurityPolicy.firstParty
+          // check if host looks like a host (instead of e.g blob: data: etc.)
+          if host.match(/[^\.]+\.[^\.]+/)
+            li: a(href=`http://${host}`)= host
+          else
+            li= host
+
+      p The website has whitelisted #{hosts.contentSecurityPolicy.firstParty.length} first-party domains and mechanisms.
+    else
+      p No CSP meta data was found. Consequently, no restrictions apply.
+
     h2(id="sec:use-of-social-media") Use of Social Media and Collaboration Platforms
 
     if links.social.length > 0    
@@ -325,6 +343,21 @@ html(xmlns='http://www.w3.org/1999/xhtml', xml:lang='en', lang='en')
     else
       p No third-party web beacons were found.
 
+    if hosts.contentSecurityPolicy.firstParty.length > 0
+
+      h4 Third-Party Content Security Policy Hosts
+
+      p Upon browser request of a web page, websites can indicate in the #[a(href="https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP") Content Security Policy] (CSP) meta data sent along with the requested web page a whitelist of mechanisms, domains and subdomains that browsers must respect when embedding components, such as styles, fonts, beacons, videos, maps, etc.
+
+      ol
+        each host in hosts.contentSecurityPolicy.thirdParty
+          li: a(href=`http://${host}`)= host
+
+      if hosts.contentSecurityPolicy.thirdParty.length > 0
+        p The website has whitelisted #{hosts.contentSecurityPolicy.thirdParty.length} distinct third-party hosts.
+      else
+        p No third-party content security policy hosts were whitelisted.
+
     h4(id="sec:unsecure-forms") Web Forms with non-encrypted Transmission
 
     if unsafeForms.length > 0

collector/browser-session.js

@@ -5,6 +5,7 @@ const url = require("url");
 const escapeRegExp = require("lodash/escapeRegExp");
 const got = require("got");
 const sampleSize = require("lodash/sampleSize");
+const parseContentSecurityPolicy = require("content-security-policy-parser").default;
 
 const UserAgent =
   "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.96 Safari/537.36";
@@ -121,6 +122,10 @@ async function createBrowserSession(browser_args, browser_logger) {
         firstParty: new Set(),
         thirdParty: new Set(),
       },
+      contentSecurityPolicy: {
+        firstParty: new Set(),
+        thirdParty: new Set(),
+      }
     };
 
     // record all requested hosts
@@ -135,6 +140,31 @@ async function createBrowserSession(browser_args, browser_logger) {
         }
       }
     });
+
+    page.on("response", (response) => {
+      const l = url.parse(response.url());
+      // WEC only records CSPs from first-party responses
+      if (isFirstParty(refs_regexp, l)) {
+        const csp = response.headers()['content-security-policy'];
+        if(csp) {
+          parseContentSecurityPolicy(csp)
+          .forEach((hostnames) => {
+            hostnames.forEach((hostname) => {
+              let match = hostname.match(/[^:\/']+$/) // strip possible schemas and exclude quoted words
+              if (match == null || match.length == 0 || match[0].match(refs_regexp)) {
+                if(hostname.startsWith('\'nonce-')) {
+                  hosts.contentSecurityPolicy.firstParty.add('\'nonce-...\'');
+                } else {
+                  hosts.contentSecurityPolicy.firstParty.add(hostname);
+                }
+              } else {
+                hosts.contentSecurityPolicy.thirdParty.add(hostname);
+              }
+            });
+          });
+        }
+      }
+    });
 
     // set predefined cookies if any
     set_cookies(page, output.uri_ins, args, output, logger);

inspector/index.js

@@ -223,6 +223,7 @@ async function inspector(args, logger, pageSession, output) {
       cookies: arrayFromParties(c.pageSession.hosts.cookies),
       localStorage: arrayFromParties(c.pageSession.hosts.localStorage),
       links: arrayFromParties(c.pageSession.hosts.links),
+      contentSecurityPolicy: arrayFromParties(c.pageSession.hosts.contentSecurityPolicy),
     };
   };
 

package.json

@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "@cliqz/adblocker-puppeteer": "^1.25.2",
+    "content-security-policy-parser": "^0.6.0",
     "fs-extra": "^11.1.0",
     "git-describe": "^4.0.4",
     "github-markdown-css": "^5.2.0",