feat: added IPv6 support to web and api, added ctx.allowlistValue for…

… IPv4/IPv6 reverse client hostname lookup, fixed linting for node preferred protocol
forwardemail · Dec 6, 2023 · ddbba52 · ddbba52
1 parent cf55295
commit ddbba52
Show file tree

Hide file tree

Showing 88 changed files with 264 additions and 179 deletions.
diff --git a/.env.defaults b/.env.defaults
@@ -41,7 +41,7 @@ NODE_ENV=development
 #################
 ### rate limit ##
 #################
-RATELIMIT_ALLOWLIST=127.0.0.1
+RATELIMIT_ALLOWLIST=localhost,127.0.0.1,::1
 
 ###########
 ## proxy ##

diff --git a/.xo-config.js b/.xo-config.js
@@ -11,7 +11,6 @@ module.exports = {
     'no-warning-comments': 'off',
     'unicorn/prefer-top-level-await': 'off',
     'unicorn/no-process-exit': 'off',
-    'unicorn/prefer-node-protocol': 'off',
     'max-depth': 'off'
   },
   overrides: [
@@ -21,6 +20,7 @@ module.exports = {
       plugins: ['compat', 'no-smart-quotes'],
       rules: {
         'compat/compat': 'error',
+        'unicorn/prefer-node-protocol': 'off',
         'no-smart-quotes/no-smart-quotes': 'error',
         'n/prefer-global/process': 'off',
         'import/no-unassigned-import': 'off'

diff --git a/ansible-playbook.js b/ansible-playbook.js
@@ -3,10 +3,10 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const process = require('process');
-const fs = require('fs');
-const path = require('path');
-const { execSync } = require('child_process');
+const process = require('node:process');
+const fs = require('node:fs');
+const path = require('node:path');
+const { execSync } = require('node:child_process');
 
 const parse = require('parse-git-config');
 

diff --git a/api.js b/api.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const process = require('process');
+const process = require('node:process');
 
 // eslint-disable-next-line import/no-unassigned-import
 require('#config/env');
@@ -33,9 +33,7 @@ monitorServer();
 
 (async () => {
   try {
-    // TODO: hard-coded until we get authbind in ansible setup
-    // TODO: and also until we use `ctx.ip` with reverse lookup for hostname/root match in allowlist env file
-    await api.listen(api.config.port, '0.0.0.0');
+    await api.listen(api.config.port);
     if (process.send) process.send('ready');
     const { port } = api.server.address();
     logger.info(

diff --git a/app/controllers/api/v1/paypal.js b/app/controllers/api/v1/paypal.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const { promisify } = require('util');
+const { promisify } = require('node:util');
 
 const Boom = require('@hapi/boom');
 const _ = require('lodash');

diff --git a/app/controllers/api/v1/self-test.js b/app/controllers/api/v1/self-test.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const os = require('os');
+const os = require('node:os');
 
 const Boom = require('@hapi/boom');
 const _ = require('lodash');

diff --git a/app/controllers/web/api.js b/app/controllers/web/api.js
@@ -3,9 +3,9 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const childProcess = require('child_process');
-const path = require('path');
-const util = require('util');
+const childProcess = require('node:child_process');
+const path = require('node:path');
+const util = require('node:util');
 
 const Meta = require('koa-meta');
 const RE2 = require('re2');

diff --git a/app/controllers/web/guides.js b/app/controllers/web/guides.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('path');
+const path = require('node:path');
 
 const Meta = require('koa-meta');
 const pug = require('pug');

diff --git a/app/models/payments.js b/app/models/payments.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('path');
+const path = require('node:path');
 
 const Boom = require('@hapi/boom');
 const numeral = require('numeral');

diff --git a/app/views/_nav.pug b/app/views/_nav.pug
@@ -275,7 +275,7 @@ nav.navbar(class=navbarClasses.join(" "))
 
           //- Resources
           if !isBot(ctx.get('User-Agent'))
-            - const resourcePages = ["/encrypted-email", "/private-business-email", "/faq", "/help", "/disposable-addresses", "/domain-registration", "/reserved-email-addresses", "/denylist-removal"];
+            - const resourcePages = ["/encrypted-email", "/private-business-email", "/faq", "/help", "/disposable-addresses", "/domain-registration", "/reserved-email-addresses", "/denylist"];
             - const companyPages = ["/about", "/privacy", "/terms", "/report-abuse"];
             - const isResourcesPage = (ctx.pathWithoutLocale === "/resources" || resourcePages.includes(ctx.pathWithoutLocale) || companyPages.includes(ctx.pathWithoutLocale)) && !hasPricingLink;
             li.nav-item.dropdown.no-js
@@ -306,6 +306,8 @@ nav.navbar(class=navbarClasses.join(" "))
                       )
                         if resourcePage === '/faq'
                           = t("Frequently Asked Questions")
+                        else if resourcePage === '/denylist'
+                          = t("Denylist Removal")
                         else
                           = t(titleize(humanize(resourcePage.replace("/", ""))))
                   hr.dropdown-divider

diff --git a/config/api.js b/config/api.js
@@ -3,13 +3,20 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
+const ipaddr = require('ipaddr.js');
+const isFQDN = require('is-fqdn');
 const sharedConfig = require('@ladjs/shared-config');
 
 const routes = require('../routes');
+
+const env = require('./env');
+
 const config = require('.');
+
+const createTangerine = require('#helpers/create-tangerine');
 const i18n = require('#helpers/i18n');
 const logger = require('#helpers/logger');
-const createTangerine = require('#helpers/create-tangerine');
+const parseRootDomain = require('#helpers/parse-root-domain');
 
 const sharedAPIConfig = sharedConfig('API');
 
@@ -28,6 +35,42 @@ module.exports = {
       app.context.client,
       app.context.logger
     );
+    app.use(async (ctx, next) => {
+      // convert local IPv6 addresses to IPv4 format
+      // <https://blog.apify.com/ipv4-mapped-ipv6-in-nodejs/>
+      if (ipaddr.isValid(ctx.request.ip)) {
+        const addr = ipaddr.parse(ctx.request.ip);
+        if (addr.kind() === 'ipv6' && addr.isIPv4MappedAddress())
+          ctx.request.ip = addr.toIPv4Address().toString();
+      }
+
+      // if we need to allowlist certain IP which resolve to our hostnames
+      if (ctx.resolver) {
+        try {
+          // maximum of 3s before ac times out
+          const abortController = new AbortController();
+          const timeout = setTimeout(() => abortController.abort(), 3000);
+          const [clientHostname] = await ctx.resolver.reverse(
+            ctx.request.ip,
+            abortController
+          );
+          clearTimeout(timeout);
+          if (isFQDN(clientHostname)) {
+            if (env.RATELIMIT_ALLOWLIST.includes(clientHostname))
+              ctx.allowlistValue = clientHostname;
+            else {
+              const rootClientHostname = parseRootDomain(clientHostname);
+              if (env.RATELIMIT_ALLOWLIST.includes(rootClientHostname))
+                ctx.allowlistValue = rootClientHostname;
+            }
+          }
+        } catch (err) {
+          ctx.logger.warn(err);
+        }
+      }
+
+      return next();
+    });
   },
   bodyParserIgnoredPathGlobs: ['/v1/log', '/v1/emails']
 };
diff --git a/config/bree.js b/config/bree.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('path');
+const path = require('node:path');
 
 module.exports = {
   root: path.join(__dirname, '..', 'jobs')

diff --git a/config/cookies.js b/config/cookies.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const process = require('process');
+const process = require('node:process');
 
 module.exports = {
   // <https://github.com/pillarjs/cookies#cookiesset-name--value---options-->

diff --git a/config/env.js b/config/env.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('path');
+const path = require('node:path');
 
 // eslint-disable-next-line n/prefer-global/process
 const test = process.env.NODE_ENV === 'test';

diff --git a/config/i18n.js b/config/i18n.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('path');
+const path = require('node:path');
 
 const locales = require('./locales');
 const phrases = require('./phrases');

diff --git a/config/index.js b/config/index.js
@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('path');
-const os = require('os');
+const path = require('node:path');
+const os = require('node:os');
 
 const Axe = require('axe');
 const Boom = require('@hapi/boom');
@@ -97,6 +97,7 @@ const config = {
   // custom rate limiting lookup for allowing whitelisted customers
   rateLimit: {
     id(ctx) {
+      if (ctx.allowlistValue) return false;
       if (typeof ctx.isAuthenticated !== 'function' || !ctx.isAuthenticated())
         return ctx.ip;
       // return `false` if the user is whitelisted

diff --git a/config/koa-cash.js b/config/koa-cash.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const { Buffer } = require('buffer');
+const { Buffer } = require('node:buffer');
 
 const ms = require('ms');
 const pTimeout = require('p-timeout');

diff --git a/config/payments.js b/config/payments.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const process = require('process');
+const process = require('node:process');
 
 const env = require('./env');
 

diff --git a/config/web.js b/config/web.js
@@ -3,24 +3,30 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const process = require('process');
+const process = require('node:process');
 
 const Boom = require('@hapi/boom');
+const ipaddr = require('ipaddr.js');
+const isFQDN = require('is-fqdn');
 const isSANB = require('is-string-and-not-blank');
 const mongoose = require('mongoose');
 const ms = require('ms');
 const sharedConfig = require('@ladjs/shared-config');
 const { Octokit } = require('@octokit/core');
 
 const routes = require('../routes');
-const env = require('./env');
+
 const cookieOptions = require('./cookies');
+const env = require('./env');
 const koaCashConfig = require('./koa-cash');
+
 const config = require('.');
+
+const createTangerine = require('#helpers/create-tangerine');
 const i18n = require('#helpers/i18n');
 const isErrorConstructorName = require('#helpers/is-error-constructor-name');
 const logger = require('#helpers/logger');
-const createTangerine = require('#helpers/create-tangerine');
+const parseRootDomain = require('#helpers/parse-root-domain');
 
 const octokit = new Octokit({
   auth: env.GITHUB_OCTOKIT_TOKEN
@@ -198,10 +204,44 @@ module.exports = (redis) => ({
       app.context.client,
       app.context.logger
     );
-    app.use((ctx, next) => {
+    app.use(async (ctx, next) => {
       // since we're on an older helmet version due to koa-helmet
       // <https://github.com/helmetjs/helmet/issues/230>
       ctx.set('X-XSS-Protection', '0');
+
+      // convert local IPv6 addresses to IPv4 format
+      // <https://blog.apify.com/ipv4-mapped-ipv6-in-nodejs/>
+      if (ipaddr.isValid(ctx.request.ip)) {
+        const addr = ipaddr.parse(ctx.request.ip);
+        if (addr.kind() === 'ipv6' && addr.isIPv4MappedAddress())
+          ctx.request.ip = addr.toIPv4Address().toString();
+      }
+
+      // if we need to allowlist certain IP which resolve to our hostnames
+      if (ctx.resolver) {
+        try {
+          // maximum of 3s before ac times out
+          const abortController = new AbortController();
+          const timeout = setTimeout(() => abortController.abort(), 3000);
+          const [clientHostname] = await ctx.resolver.reverse(
+            ctx.request.ip,
+            abortController
+          );
+          clearTimeout(timeout);
+          if (isFQDN(clientHostname)) {
+            if (env.RATELIMIT_ALLOWLIST.includes(clientHostname))
+              ctx.allowlistValue = clientHostname;
+            else {
+              const rootClientHostname = parseRootDomain(clientHostname);
+              if (env.RATELIMIT_ALLOWLIST.includes(rootClientHostname))
+                ctx.allowlistValue = rootClientHostname;
+            }
+          }
+        } catch (err) {
+          ctx.logger.warn(err);
+        }
+      }
+
       return next();
     });
   },

diff --git a/gulpfile.js b/gulpfile.js
@@ -3,9 +3,9 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const process = require('process');
-const path = require('path');
-const fs = require('fs');
+const process = require('node:process');
+const path = require('node:path');
+const fs = require('node:fs');
 
 // required to disable watching of I18N files in @ladjs/i18n
 // otherwises tasks will fail to exit due to watchers running

diff --git a/helpers/encrypt-decrypt.js b/helpers/encrypt-decrypt.js
@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const crypto = require('crypto');
-const { Buffer } = require('buffer');
+const crypto = require('node:crypto');
+const { Buffer } = require('node:buffer');
 
 const env = require('#config/env');
 

diff --git a/helpers/paypal.js b/helpers/paypal.js
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const { promisify } = require('util');
+const { promisify } = require('node:util');
 
 const paypal = require('paypal-rest-sdk');
 const superagent = require('superagent');

diff --git a/jobs/account-updates.js b/jobs/account-updates.js
@@ -6,9 +6,9 @@
 // eslint-disable-next-line import/no-unassigned-import
 require('#config/env');
 
-const process = require('process');
-const os = require('os');
-const { parentPort } = require('worker_threads');
+const process = require('node:process');
+const os = require('node:os');
+const { parentPort } = require('node:worker_threads');
 
 // eslint-disable-next-line import/no-unassigned-import
 require('#config/mongoose');