chore(platform): Extract relay into separate services (#2823)

* Split relay into multiple services - no GHA workflows yet * lint errors * Fixes and GHA workflows * Corrects old masked email pattern * Error, check utils, source account retrieval tweaks and worker renames * Add missing lock file from renames in prev commit * Cleanup of types and added eslint * yarn :/ * More eslint * Rename workflow and env vars from relay to email.
proofzero · Jan 28, 2024 · 5ca7b8c · 5ca7b8c
1 parent d7fb632
commit 5ca7b8c
Show file tree

Hide file tree

Showing 43 changed files with 964 additions and 96 deletions.
diff --git a/.github/workflows/main-core.yaml b/.github/workflows/main-core.yaml
@@ -61,7 +61,7 @@ jobs:
             SECRET_JWKS
 
             INTERNAL_DKIM_SELECTOR
-
+            INTERNAL_EMAIL_DISTRIBUTION_KEY
             SECRET_RELAY_DKIM_PRIVATE_KEY
 
             INTERNAL_GOOGLE_OAUTH_CLIENT_ID
@@ -94,6 +94,7 @@ jobs:
 
           INTERNAL_DKIM_SELECTOR: ${{ secrets.INTERNAL_DKIM_SELECTOR }}
 
+          INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_DEV}}
           SECRET_RELAY_DKIM_PRIVATE_KEY: ${{ secrets.SECRET_RELAY_DKIM_PRIVATE_KEY }}
 
           INTERNAL_GOOGLE_OAUTH_CLIENT_ID: ${{ vars.INTERNAL_GOOGLE_OAUTH_CLIENT_ID_DEV }}

diff --git a/.github/workflows/main-emaildistributor.yaml b/.github/workflows/main-emaildistributor.yaml
@@ -0,0 +1,51 @@
+name: Singleton Email distributor
+
+on:
+  push:
+    branches:
+      - main
+
+defaults:
+  run:
+    working-directory: platform/emaildistributor
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: dev
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: cachix/install-nix-action@v18
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - run: nix-build ../platform.nix
+
+      - name: Cache Dependencies
+        id: cache-modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            node_modules
+            .yarn
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Dependencies
+        run: yarn install
+
+      - name: Test
+        run: yarn run test
+
+      - name: Deploy
+        uses: cloudflare/[email protected]
+        with:
+          wranglerVersion: '3.19.0'
+          apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }}
+          accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }}
+          workingDirectory: platform/emaildistributor
+          command: publish
+          secrets: |
+            SECRET_EMAIL_DISTRIBUTION_MAP
+        env:
+          SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }}
diff --git a/.github/workflows/main-emailinbounder.yaml b/.github/workflows/main-emailinbounder.yaml
@@ -0,0 +1,53 @@
+name: Mail Email inbounder
+
+on:
+  push:
+    branches:
+      - main
+
+defaults:
+  run:
+    working-directory: platform/emailinbounder
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: dev
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: cachix/install-nix-action@v18
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - run: nix-build ../platform.nix
+
+      - name: Cache Dependencies
+        id: cache-modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            node_modules
+            .yarn
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Dependencies
+        run: yarn install
+
+      - name: Test
+        run: yarn run test
+
+      - name: Deploy
+        uses: cloudflare/[email protected]
+        with:
+          wranglerVersion: '3.19.0'
+          apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }}
+          accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }}
+          workingDirectory: platform/emailinbounder
+          command: publish
+          secrets: |
+            SECRET_EMAIL_DISTRIBUTION_MAP
+            INTERNAL_EMAIL_DISTRIBUTION_KEY
+        env:
+          SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }}
+          INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_DEV }}
diff --git a/.github/workflows/next-core.yaml b/.github/workflows/next-core.yaml
@@ -61,7 +61,7 @@ jobs:
             SECRET_JWKS
 
             INTERNAL_DKIM_SELECTOR
-
+            INTERNAL_EMAIL_DISTRIBUTION_KEY
             SECRET_RELAY_DKIM_PRIVATE_KEY
 
             INTERNAL_GOOGLE_OAUTH_CLIENT_ID
@@ -93,6 +93,7 @@ jobs:
           SECRET_JWKS: ${{ secrets.SECRET_JWKS_TEST }}
 
           INTERNAL_DKIM_SELECTOR: ${{ secrets.INTERNAL_DKIM_SELECTOR }}
+          INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_TEST }}
 
           SECRET_RELAY_DKIM_PRIVATE_KEY: ${{ secrets.SECRET_RELAY_DKIM_PRIVATE_KEY }}
 

diff --git a/.github/workflows/next-emailinbounder.yaml b/.github/workflows/next-emailinbounder.yaml
@@ -0,0 +1,55 @@
+name: Next Email inbounder
+
+on:
+  push:
+    tags:
+      - '*'
+
+defaults:
+  run:
+    working-directory: platform/emailinbounder
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: next
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: cachix/install-nix-action@v18
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - run: nix-build ../platform.nix
+
+      - name: Cache Dependencies
+        id: cache-modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            node_modules
+            .yarn
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Dependencies
+        run: yarn install
+
+      - name: Test
+        run: yarn run test
+
+      - name: Deploy
+        uses: cloudflare/[email protected]
+        with:
+          wranglerVersion: '3.19.0'
+          apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }}
+          accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }}
+          workingDirectory: platform/emailinbounder
+          command: publish --config wrangler.next.toml --env next
+          environment: 'next'
+          secrets: |
+            SECRET_EMAIL_DISTRIBUTION_MAP
+            INTERNAL_EMAIL_DISTRIBUTION_KEY
+
+        env:
+          SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }}
+          INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_TEST }}
diff --git a/.github/workflows/release-core.yaml b/.github/workflows/release-core.yaml
@@ -60,7 +60,7 @@ jobs:
             SECRET_JWKS
 
             INTERNAL_DKIM_SELECTOR
-
+            INTERNAL_EMAIL_DISTRIBUTION_KEY
             SECRET_RELAY_DKIM_PRIVATE_KEY
 
             INTERNAL_GOOGLE_OAUTH_CLIENT_ID
@@ -92,6 +92,7 @@ jobs:
           SECRET_JWKS: ${{ secrets.SECRET_JWKS_PROD }}
 
           INTERNAL_DKIM_SELECTOR: ${{ secrets.INTERNAL_DKIM_SELECTOR }}
+          INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_PROD }}
 
           SECRET_RELAY_DKIM_PRIVATE_KEY: ${{ secrets.SECRET_RELAY_DKIM_PRIVATE_KEY }}
 

diff --git a/.github/workflows/release-emailinbounder.yaml b/.github/workflows/release-emailinbounder.yaml
@@ -0,0 +1,54 @@
+name: Release Email inbounder
+
+on:
+  release:
+    types: [published]
+
+defaults:
+  run:
+    working-directory: platform/emailinbounder
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: prod
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: cachix/install-nix-action@v18
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - run: nix-build ../platform.nix
+
+      - name: Cache Dependencies
+        id: cache-modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            node_modules
+            .yarn
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Dependencies
+        run: yarn install
+
+      - name: Test
+        run: yarn run test
+
+      - name: Deploy
+        uses: cloudflare/[email protected]
+        with:
+          wranglerVersion: '3.19.0'
+          apiToken: ${{ secrets.TOKEN_CLOUDFLARE_API }}
+          accountId: ${{ secrets.INTERNAL_CLOUDFLARE_ACCOUNT_ID }}
+          workingDirectory: platform/emailinbounder
+          command: publish --config wrangler.current.toml --env current
+          environment: 'current'
+          secrets: |
+            SECRET_EMAIL_DISTRIBUTION_MAP
+            INTERNAL_EMAIL_DISTRIBUTION_KEY
+
+        env:
+          SECRET_EMAIL_DISTRIBUTION_MAP: ${{ secrets.SECRET_EMAIL_DISTRIBUTION_MAP }}
+          INTERNAL_EMAIL_DISTRIBUTION_KEY: ${{secrets.INTERNAL_EMAIL_DISTRIBUTION_KEY_PROD }}
diff --git a/.yarn/cache/@cloudflare-workers-types-npm-4.20221111.1-69d0ea0925-6ee1ba28ee.zip b/.yarn/cache/@cloudflare-workers-types-npm-4.20221111.1-69d0ea0925-6ee1ba28ee.zip
diff --git a/packages/types/email.ts b/packages/types/email.ts
@@ -2,3 +2,15 @@ export enum ReconciliationNotificationType {
   Billing = 'BILLING',
   Dev = 'DEV',
 }
+
+/** CF EmailMessage type; not provided in CF types lib */
+export interface CloudflareEmailMessage {
+  readonly from: string
+  readonly to: string
+  readonly headers: Headers
+  readonly raw: ReadableStream
+  readonly rawSize: number
+
+  setReject(reason: string): void
+  forward(rcptTo: string, headers?: Headers): Promise<void>
+}
diff --git a/platform/account/src/jsonrpc/methods/getMaskedAddress.ts b/platform/account/src/jsonrpc/methods/getMaskedAddress.ts
@@ -41,5 +41,9 @@ export const getMaskedAddressMethod: GetMaskedAddressMethod = async ({
   }
 
   const node = new EmailAccount(ctx.account, ctx.env)
-  return node.getMaskedAddress(input.clientId)
+  return node.getMaskedAddress(
+    input.clientId,
+    ctx.env.INTERNAL_EMAIL_DISTRIBUTION_KEY,
+    ctx.env.INTERNAL_RELAY_DKIM_DOMAIN
+  )
 }
diff --git a/platform/account/src/jsonrpc/methods/getSourceFromMaskedAddress.ts b/platform/account/src/jsonrpc/methods/getSourceFromMaskedAddress.ts
@@ -0,0 +1,63 @@
+import { z } from 'zod'
+import { router } from '@proofzero/platform.core'
+import { EDGE_ACCOUNT } from '@proofzero/platform.account/src/constants'
+import { Context } from '../../context'
+import { EmailAccountType } from '@proofzero/types/account'
+import { AccountURN, AccountURNSpace } from '@proofzero/urns/account'
+import {
+  BadRequestError,
+  InternalServerError,
+  NotFoundError,
+} from '@proofzero/errors'
+import { generateHashedIDRef } from '@proofzero/packages/urns/idref'
+import { EmailAccount, initAccountNodeByName } from '../../nodes'
+
+export const GetSourceByMaskedAddressInput = z.object({
+  maskedEmail: z.string(),
+})
+
+export const GetSourceByMaskedAddressOutput = z.object({
+  nickname: z.string(),
+  sourceEmail: z.string(),
+})
+
+type GetSourceByMaskedAddressParams = z.infer<
+  typeof GetSourceByMaskedAddressInput
+>
+type GetSourceByMaskedAddressResult = z.infer<
+  typeof GetSourceByMaskedAddressOutput
+>
+
+export const getSourceFromMaskedAddressMethod = async ({
+  input,
+  ctx,
+}: {
+  input: GetSourceByMaskedAddressParams
+  ctx: Context
+}): Promise<GetSourceByMaskedAddressResult> => {
+  const nss = generateHashedIDRef(EmailAccountType.Mask, input.maskedEmail)
+  const urn = AccountURNSpace.componentizedUrn(nss)
+  const node = initAccountNodeByName(urn, ctx.env.Account)
+
+  const emailNode = new EmailAccount(node, ctx.env)
+  const sourceAccountURN = await emailNode.getSourceAccount()
+  if (!sourceAccountURN)
+    throw new NotFoundError({
+      message: `Could not find hidden address ${input.maskedEmail}`,
+    })
+
+  const sourceAccountNode = initAccountNodeByName(
+    sourceAccountURN,
+    ctx.env.Account
+  )
+
+  const name = (await sourceAccountNode.class.getNickname()) || ''
+  const address = await sourceAccountNode.class.getAddress()
+
+  if (!address)
+    throw new InternalServerError({
+      message: `Could not find source address for masked email ${input.maskedEmail}`,
+    })
+
+  return { sourceEmail: address, nickname: name }
+}
diff --git a/platform/account/src/jsonrpc/router.ts b/platform/account/src/jsonrpc/router.ts
@@ -147,6 +147,11 @@ import {
   SetSourceAccountInput,
   SetSourceAccountOutput,
 } from './methods/setSourceAccount'
+import {
+  GetSourceByMaskedAddressInput,
+  GetSourceByMaskedAddressOutput,
+  getSourceFromMaskedAddressMethod,
+} from './methods/getSourceFromMaskedAddress'
 
 const t = initTRPC.context<Context>().create({ errorFormatter })
 
@@ -403,4 +408,10 @@ export const appRouter = t.router({
     .input(SetSourceAccountInput)
     .output(SetSourceAccountOutput)
     .mutation(setSourceAccountMethod),
+  getSourceFromMaskedAddress: t.procedure
+    .use(LogUsage)
+    .use(Analytics)
+    .input(GetSourceByMaskedAddressInput)
+    .output(GetSourceByMaskedAddressOutput)
+    .query(getSourceFromMaskedAddressMethod),
 })