feat: make cosmo usable with aws s3

cdn-server/.env.example

@@ -1,4 +1,9 @@
 PORT=11000
 AUTH_JWT_SECRET=fkczyomvdprgvtmvkuhvprxuggkbgwld
 AUTH_ADMISSION_JWT_SECRET="uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW"
-S3_STORAGE_URL=http://minio:changeme@localhost:10000/cosmo
+
+S3_STORAGE_URL=http://minio:changeme@localhost:10000/cosmo
+S3_REGION='auto'
+S3_ENDPOINT=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=

cdn-server/package.json

@@ -25,6 +25,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.529.1",
     "@hono/node-server": "1.11.0",
+    "@wundergraph/cosmo-shared": "workspace:*",
     "@wundergraph/cosmo-cdn": "workspace:*",
     "dotenv": "^16.4.5",
     "hono": "4.2.7"

cdn-server/src/s3.ts

@@ -1,4 +1,5 @@
 import { GetObjectCommand, HeadObjectCommand, NoSuchKey, NotFound, S3Client } from '@aws-sdk/client-s3';
+import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared';
 import { BlobNotFoundError, BlobObject, BlobStorage } from '@wundergraph/cosmo-cdn';
 import { Context } from 'hono';
 
@@ -86,16 +87,20 @@ class S3BlobStorage implements BlobStorage {
 
 export const createS3BlobStorage = (storageUrl: string): BlobStorage => {
   const url = new URL(storageUrl);
-  const region = url.searchParams.get('region') ?? 'default';
-  const s3Client = new S3Client({
+  const region = url.searchParams.get('region') ?? process.env.S3_REGION ?? 'default';
+  const endpoint = url.searchParams.get('endpoint') ?? process.env.S3_ENDPOINT;
+  const username = process.env.S3_ACCESS_KEY_ID || '';
+  const password = process.env.S3_SECRET_ACCESS_KEY || '';
+
+  const bucketName = extractS3BucketName(storageUrl);
+  const s3Config = createS3ClientConfig(bucketName, {
+    url: storageUrl,
     region,
-    endpoint: url.origin,
-    credentials: {
-      accessKeyId: url.username ?? '',
-      secretAccessKey: url.password ?? '',
-    },
-    forcePathStyle: true,
+    endpoint,
+    username,
+    password,
   });
-  const bucketName = url.pathname.slice(1);
+  const s3Client = new S3Client(s3Config);
+
   return new S3BlobStorage(s3Client, bucketName);
 };

controlplane/.env.example

@@ -32,6 +32,10 @@ WEBHOOK_SECRET=
 
 # S3
 S3_STORAGE_URL="http://minio:changeme@localhost:10000/cosmo"
+S3_REGION="auto"
+S3_ENDPOINT=""
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
 
 # Optional for Stripe Integration
 DEFAULT_PLAN="developer@1"

controlplane/src/core/build-server.ts

@@ -2,6 +2,7 @@ import Fastify, { FastifyBaseLogger } from 'fastify';
 import { S3Client } from '@aws-sdk/client-s3';
 import { fastifyConnectPlugin } from '@connectrpc/connect-fastify';
 import { cors, createContextValues } from '@connectrpc/connect';
+import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared';
 import fastifyCors from '@fastify/cors';
 import { pino, stdTimeFunctions, LoggerOptions } from 'pino';
 import { compressionBrotli, compressionGzip } from '@connectrpc/connect-node';
@@ -86,7 +87,13 @@ export interface BuildConfig {
   };
   slack: { clientID?: string; clientSecret?: string };
   cdnBaseUrl: string;
-  s3StorageUrl: string;
+  s3Storage: {
+    url: string;
+    endpoint?: string;
+    region?: string;
+    username?: string;
+    password?: string;
+  };
   mailer: {
     smtpEnabled: boolean;
     smtpHost?: string;
@@ -343,22 +350,14 @@ export default async function build(opts: BuildConfig) {
     });
   }
 
-  if (!opts.s3StorageUrl) {
+  if (!opts.s3Storage || !opts.s3Storage.url) {
     throw new Error('S3 storage URL is required');
   }
 
-  const url = new URL(opts.s3StorageUrl);
-  const s3Client = new S3Client({
-    // For AWS S3, the region can be set via the endpoint
-    region: 'auto',
-    endpoint: url.origin,
-    credentials: {
-      accessKeyId: url.username ?? '',
-      secretAccessKey: url.password ?? '',
-    },
-    forcePathStyle: true,
-  });
-  const bucketName = url.pathname.slice(1);
+  const bucketName = extractS3BucketName(opts.s3Storage.url);
+  const s3Config = createS3ClientConfig(bucketName, opts.s3Storage);
+
+  const s3Client = new S3Client(s3Config);
   const blobStorage = new S3BlobStorage(s3Client, bucketName);
 
   /**

controlplane/src/core/env.schema.ts

@@ -99,9 +99,34 @@ export const envVariables = z
     SLACK_APP_CLIENT_ID: z.string().optional(),
     SLACK_APP_CLIENT_SECRET: z.string().optional(),
     /**
-     * S3 Storage e.g. for persistent operations
+     * S3 Storage e.g. for persistent operations.
+     *
+     * S3_STORAGE_URL: The blobStorage url containing username and password (e.g.: https://username:[email protected])
+     * S3_REGION: The region to use for the S3 storage (e.g.: us-east-1, this fallbacks to auto and must be set when using aws)
+     * S3_ENDPOINT: The aws endpoint to use for the S3 storage (e.g.: s3.amazonaws.com, this fallbacks to the origin of the S3_STORAGE_URL)
+     *
+     * Examples:
+     * Minio Storage
+     * S3_STORAGE_URL="http://minio:pass@minio:9000/cosmo"
+     * S3_REGION="auto"                           # default
+     * S3_ENDPOINT=S3_STORAGE_URL.origin          # default
+     *
+     * AWS S3 Storage
+     * S3_STORAGE_URL="https://username:[email protected]"
+     * S3_REGION="us-east-1"                      # set this for amazon to your region
+     * S3_ENDPOINT="s3.amazonaws.com"             # replaces the bucket from the S3_STORAGE_URL origin or set it manually to s3.amazonaws.com
      */
     S3_STORAGE_URL: z.string(),
+    S3_ENDPOINT: z.string().optional(),
+    S3_REGION: z.string().default('auto'),
+    /**
+     * Either use:
+     *   https://username:[email protected]
+     * Or set: S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY
+     */
+    S3_ACCESS_KEY_ID: z.string().optional(),
+    S3_SECRET_ACCESS_KEY: z.string().optional(),
+
     /**
      * Email
      */

controlplane/src/core/util.ts

@@ -1,6 +1,7 @@
 import { randomFill } from 'node:crypto';
 import { HandlerContext } from '@connectrpc/connect';
 import {
+  EnumStatusCode,
   GraphQLSubscriptionProtocol,
   GraphQLWebsocketSubprotocol,
 } from '@wundergraph/cosmo-connect/dist/common/common_pb';

controlplane/src/index.ts

@@ -41,6 +41,10 @@ const {
   SLACK_APP_CLIENT_ID,
   SLACK_APP_CLIENT_SECRET,
   S3_STORAGE_URL,
+  S3_ENDPOINT,
+  S3_REGION,
+  S3_ACCESS_KEY_ID,
+  S3_SECRET_ACCESS_KEY,
   SMTP_ENABLED,
   SMTP_HOST,
   SMTP_PORT,
@@ -116,7 +120,13 @@ const options: BuildConfig = {
     clientID: SLACK_APP_CLIENT_ID,
     clientSecret: SLACK_APP_CLIENT_SECRET,
   },
-  s3StorageUrl: S3_STORAGE_URL,
+  s3Storage: {
+    url: S3_STORAGE_URL,
+    region: S3_REGION,
+    endpoint: S3_ENDPOINT,
+    username: S3_ACCESS_KEY_ID,
+    password: S3_SECRET_ACCESS_KEY,
+  },
   mailer: {
     smtpEnabled: SMTP_ENABLED,
     smtpHost: SMTP_HOST,

controlplane/test/authentication.test.ts

@@ -48,7 +48,13 @@ describe('Authentication', (ctx) => {
         clientID: '',
         clientSecret: '',
       },
-      s3StorageUrl: 'http://localhost:9000',
+      s3Storage: {
+        url: 'http://localhost:9000',
+        region: 'auto',
+        endpoint: 'localhost:9000',
+        username: 'minio',
+        password: 'changeme',
+      },
       mailer: {
         smtpEnabled: false,
         smtpHost: '',

docker-compose.full.yml

@@ -205,10 +205,14 @@ services:
       context: .
       dockerfile: cdn-server/Dockerfile
     environment:
-      - PORT=11000
-      - AUTH_JWT_SECRET=fkczyomvdprgvtmvkuhvprxuggkbgwld
-      - AUTH_ADMISSION_JWT_SECRET="uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW"
-      - S3_STORAGE_URL=http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo
+      PORT: 11000
+      NODE_ENV: development
+      AUTH_JWT_SECRET: fkczyomvdprgvtmvkuhvprxuggkbgwld
+      AUTH_ADMISSION_JWT_SECRET: uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW
+      S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo}
+      S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}}
+      S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
+      S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
     ports:
       - '11000:11000'
     networks:
@@ -270,7 +274,10 @@ services:
       KC_API_URL: 'http://keycloak:8080'
       KC_FRONTEND_URL: 'http://localhost:8080'
       PROMETHEUS_API_URL: 'http://admin:test@prometheus:9090/api/v1'
-      S3_STORAGE_URL: http://minio:changeme@minio:9000/cosmo
+      S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo}
+      S3_REGION: ${S3_REGION_CONTROLPLANE:-${S3_REGION:-auto}}
+      S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
+      S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
       CDN_BASE_URL: 'http://cdn:11000'
       REDIS_HOST: redis
       REDIS_PORT: 6379

docker-compose.yml

@@ -164,10 +164,11 @@ services:
       context: .
       dockerfile: cdn-server/Dockerfile
     environment:
-      - PORT=11000
-      - AUTH_JWT_SECRET=fkczyomvdprgvtmvkuhvprxuggkbgwld
-      - AUTH_ADMISSION_JWT_SECRET=uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW
-      - S3_STORAGE_URL=http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo
+      PORT: 11000
+      AUTH_JWT_SECRET: fkczyomvdprgvtmvkuhvprxuggkbgwld
+      AUTH_ADMISSION_JWT_SECRET: uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW
+      S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo}
+      S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}}
     ports:
       - '11000:11000'
     networks:

helm/cosmo/README.md

@@ -35,6 +35,10 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | cdn.commonLabels | object | `{}` | Add labels to all deployed resources |
+| cdn.configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url |
+| cdn.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. |
+| cdn.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. |
+| cdn.configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url |
 | cdn.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` |  |
 | clickhouse.auth.password | string | `"changeme"` |  |
 | clickhouse.auth.username | string | `"default"` |  |
@@ -64,6 +68,10 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph
 | controlplane.configuration.prometheus.port | int | `8088` | The port where metrics are exposed. Default is port 8088. |
 | controlplane.configuration.redisHost | string | `"cosmo-redis-master"` |  |
 | controlplane.configuration.redisPort | int | `6379` |  |
+| controlplane.configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url |
+| controlplane.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. |
+| controlplane.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. |
+| controlplane.configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url |
 | controlplane.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` |  |
 | controlplane.configuration.smtp | object | `{"enabled":false,"host":"smtp.postmarkapp.com","password":"","port":587,"requireTls":true,"secure":true,"username":""}` | Use this section to configure the smtp server. |
 | controlplane.configuration.smtp.enabled | bool | `false` | Enables the smtp server. Default is false. |

helm/cosmo/charts/cdn/README.md

@@ -16,7 +16,10 @@ WunderGraph Cosmo CDN
 | autoscaling.minReplicas | int | `1` |  |
 | autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
 | commonLabels | object | `{}` | Add labels to all deployed resources |
-| configuration | string | `nil` |  |
+| configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url |
+| configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. |
+| configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. |
+| configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url |
 | deploymentStrategy | object | `{}` |  |
 | existingSecret | string | `""` | Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. |
 | extraEnvVars | list | `[]` | Allows to set additional environment variables on the container. Useful for global application non-specific settings. |

helm/cosmo/charts/cdn/templates/config-map.yaml

@@ -10,3 +10,7 @@ metadata:
     {{- include "cdn.labels" . | nindent 4 }}
 data:
   port: "{{ .Values.service.port }}"
+  s3Region: "{{ .Values.configuration.s3Region }}"
+  {{- if .Values.configuration.s3Endpoint }}
+  s3Endpoint: "{{ .Values.configuration.s3Endpoint }}"
+  {{- end }}

helm/cosmo/charts/cdn/templates/deployment.yaml

@@ -59,6 +59,36 @@ spec:
                   name: {{ include "cdn.fullname" . }}-configmap
                   key: port
 
+            - name: S3_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "cdn.fullname" . }}-configmap
+                  key: s3Region
+
+            {{- if .Values.configuration.s3Endpoint }}
+            - name: S3_ENDPOINT
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "cdn.fullname" . }}-configmap
+                  key: s3Endpoint
+            {{- end }}
+
+            {{- if .Values.configuration.s3AccessKeyId }}
+            - name: S3_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "cdn.secretName" . }}
+                  key: s3AccessKeyId
+            {{- end }}
+
+            {{- if .Values.configuration.s3SecretAccessKey }}
+            - name: S3_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "cdn.secretName" . }}
+                  key: s3SecretAccessKey
+            {{- end }}
+
             - name: S3_STORAGE_URL
               valueFrom:
                 secretKeyRef:

helm/cosmo/charts/cdn/templates/secret.yaml

@@ -17,4 +17,10 @@ stringData:
   authJwtSecret: "{{ .Values.global.controlplane.jwtSecret }}"
   authAdmissionJwtSecret: "{{ .Values.global.controlplane.admissionJwtSecret }}"
   s3StorageUrl: "{{ .Values.configuration.s3StorageUrl }}"
+  {{- if .Values.configuration.s3AccessKeyId }}
+  s3AccessKeyId: "{{ .Values.configuration.s3AccessKeyId }}"
+  {{- end }}
+  {{- if .Values.configuration.s3SecretAccessKey }}
+  s3SecretAccessKey: "{{ .Values.configuration.s3SecretAccessKey }}"
+  {{- end }}
 {{- end }}

helm/cosmo/charts/cdn/values.yaml

@@ -144,6 +144,14 @@ probes:
 #############################
 
 configuration:
+  # -- The region where the S3 bucket is located.
+  s3Region: 'auto'
+  # -- The endpoint of the S3 bucket.
+  s3Endpoint: ''
+  # -- s3 access key id, can be used instead of [username]:[password] in the url
+  s3AccessKeyId: ''
+  # -- s3 secret access key, can be used instead of [username]:[password] in the url
+  s3SecretAccessKey: ''
 
 # -- Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret.
 existingSecret: ""

helm/cosmo/charts/controlplane/README.md

@@ -49,6 +49,10 @@ WunderGraph Cosmo Controlplane
 | configuration.redisTlsCa | string | `""` | When connecting to a redis instance over TLS. Accept a cert in PEM format (as one-line with \n) or file. |
 | configuration.redisTlsCert | string | `""` |  |
 | configuration.redisTlsKey | string | `""` |  |
+| configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url |
+| configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. |
+| configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. |
+| configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url |
 | configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` |  |
 | configuration.slackAppClientId | string | `""` |  |
 | configuration.slackAppClientSecret | string | `""` |  |

helm/cosmo/charts/controlplane/templates/config-map.yaml

@@ -40,4 +40,8 @@ data:
   prometheusHost: "{{ .Values.configuration.prometheus.host }}"
   prometheusPort: "{{ .Values.configuration.prometheus.port }}"
   prometheusPath: "{{ .Values.configuration.prometheus.path }}"
-  {{ end }}
+  {{- end }}
+  s3Region: "{{ .Values.configuration.s3Region }}"
+  {{- if .Values.configuration.s3Endpoint }}
+  s3Endpoint: "{{ .Values.configuration.s3Endpoint }}"
+  {{ end }}