Merge branch 'docker' into develop

.github/workflows/build_server.yml

@@ -2,27 +2,166 @@ name: Build server
 
 on:
   push:
-    branches: [main]
-  pull_request:
-    branches: [develop, main]
+    branches: [docker, docker-prod, develop]
 
 jobs:
-  build:
+  prepare-env:
     runs-on: ubuntu-latest
+    outputs:
+      env_tag: ${{ steps.set-tag.outputs.ENV_TAG }}
+      compose_file: ${{ steps.set-tag.outputs.COMPOSE_FILE }}
+      stack_name: ${{ steps.set-tag.outputs.STACK_NAME }}
+    steps:
+      - name: Set environment tag
+        id: set-tag
+        run: |
+          if [[ ${{ github.ref }} == 'refs/heads/docker-prod' ]]; then
+            echo "ENV_TAG=prod" >> $GITHUB_OUTPUT
+            echo "COMPOSE_FILE=docker-compose.prod.yml" >> $GITHUB_OUTPUT
+            echo "STACK_NAME=prod" >> $GITHUB_OUTPUT
+          else
+            echo "ENV_TAG=staging" >> $GITHUB_OUTPUT
+            echo "COMPOSE_FILE=docker-compose.staging.yml" >> $GITHUB_OUTPUT
+            echo "STACK_NAME=staging" >> $GITHUB_OUTPUT
+          fi
+  build-and-push-images:
+    needs: prepare-env
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - image: server
+            dockerfile: ./packages/server/src/Dockerfile
+          - image: stage-transcriptions
+            dockerfile: ./packages/server/workers/stage-transcriptions/Dockerfile
+          - image: session-transcriptions
+            dockerfile: ./packages/server/workers/session-transcriptions/Dockerfile
+          - image: clips
+            dockerfile: ./packages/server/workers/clips/Dockerfile
+          - image: reel-creator
+            dockerfile: ./packages/reel-creator/Dockerfile
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Setup Node.js 20...
-        uses: actions/setup-node@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Add caching for Docker layers
+      - name: Cache Docker layers
+        uses: actions/cache@v3
         with:
-          node-version: "20"
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ matrix.image }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ matrix.image }}-
+            ${{ runner.os }}-buildx-
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: https://ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.TOKEN }}
 
-      - name: Install yarn...
-        run: npm install -g yarn
+      - name: Build and push ${{ matrix.image }}
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/streamethorg/streameth-platform/${{ matrix.image }}:${{ needs.prepare-env.outputs.env_tag }}
+            ghcr.io/streamethorg/streameth-platform/${{ matrix.image }}:${{ needs.prepare-env.outputs.env_tag }}-${{ github.sha }}
+          file: ${{ matrix.dockerfile }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
 
-      - name: Install dependencies...
-        run: yarn install:all
+      # Temp fix for cache handling
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+  deploy:
+    needs: [prepare-env, build-and-push-images]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Build server
-        run: yarn build:server
+      # Setup SSH key
+      - name: Setup SSH key
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -H 145.223.118.217 >> ~/.ssh/known_hosts
+      # Setup Docker context from base64-encoded secret
+      - name: Setup Docker context
+        run: |
+          mkdir -p ~/.docker/contexts
+          if [[ ${{ github.ref }} == 'refs/heads/docker-prod' ]]; then
+            echo "${{ secrets.DOCKER_CONTEXT_PROD_B64 }}" | base64 -d > ~/.docker/contexts/streameth-prod.tar.gz
+            docker context import streameth-prod ~/.docker/contexts/streameth-prod.tar.gz
+            docker context use streameth-prod
+          else
+            echo "${{ secrets.DOCKER_CONTEXT_STAGING_B64 }}" | base64 -d > ~/.docker/contexts/streameth-staging.tar.gz
+            docker context import streameth-staging ~/.docker/contexts/streameth-staging.tar.gz
+            docker context use streameth-staging
+          fi
+      # Copy compose files to server
+      - name: Copy compose files
+        run: |
+          scp docker-compose.*.yml [email protected]:/home/streameth/streameth/
+      # Log in to registry on deployment server
+      - name: Log in to registry on deployment server
+        run: |
+          echo "${{ secrets.TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      # Update services based on environment
+      - name: Update services
+        run: |
+          STACK_NAME="${{ needs.prepare-env.outputs.stack_name }}"
+          SHA="${{ github.sha }}"
+          
+          # Function to update a service and capture its exit status
+          update_service() {
+            local service=$1
+            local temp_file=$(mktemp)
+            if docker service update \
+              --with-registry-auth \
+              --image ghcr.io/streamethorg/streameth-platform/$service:${{ needs.prepare-env.outputs.env_tag }}-$SHA \
+              ${STACK_NAME}_${service} > $temp_file 2>&1; then
+              echo "✅ Service ${STACK_NAME}_${service} updated successfully"
+              rm $temp_file
+              return 0
+            else
+              echo "❌ Failed to update ${STACK_NAME}_${service}"
+              cat $temp_file
+              rm $temp_file
+              return 1
+            fi
+          }
+          
+          # Start all updates in parallel and capture PIDs
+          pids=()
+          for service in server stage-transcriptions session-transcriptions clips reel-creator; do
+            update_service $service &
+            pids+=($!)
+          done
+          
+          # Wait for all updates and check for failures
+          failed=0
+          for pid in ${pids[@]}; do
+            if ! wait $pid; then
+              failed=1
+            fi
+          done
+          
+          # Exit with failure if any update failed
+          exit $failed
+      # Deploy router if it doesn't exist (only needs to be done once)
+      - name: Deploy router if needed
+        if: github.ref == 'refs/heads/docker-prod'  # Only check/deploy router on prod branch pushes
+        run: |
+          if ! docker stack ls | grep -q "router"; then
+            docker stack deploy -c /home/streameth/streameth/docker-compose.router.yml router
+          fi

.github/workflows/clean_caches.yml

@@ -0,0 +1,32 @@
+name: cleanup caches
+on:
+  workflow_dispatch:  # Adding manual trigger option
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup all caches
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For pull request events, clean specific PR branch caches
+            BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
+            echo "Cleaning caches for PR branch: $BRANCH"
+            cacheKeysForPR=$(gh cache list --ref $BRANCH --limit 100 --json id --jq '.[].id')
+          else
+            # For manual dispatch, clean all caches
+            echo "Cleaning all repository caches"
+            cacheKeysForPR=$(gh cache list --limit 100 --json id --jq '.[].id')
+          fi
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh cache delete $cacheKey --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.TOKEN }}
+          GH_REPO: ${{ github.repository }}

.gitignore

@@ -1,5 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+undefined/*
+undefined
 # dependencies
 node_modules
 /.pnp
@@ -55,3 +57,7 @@ google_sa_secret.json
 .yarn/*
 .yarn
 .yarn/
+
+
+*.log
+*.mp4

.husky/pre-commit

@@ -1,3 +1,2 @@
 #!/usr/bin/env bash
 
-yarn lint-staged --verbose

docker-compose-dev.yml

@@ -0,0 +1,148 @@
+version: '3'
+
+# Define common configurations that can be reused
+x-common-environment: &common-environment
+  NODE_ENV: ${NODE_ENV}
+  APP_PORT: ${APP_PORT}
+  BASE_URL: ${BASE_URL}
+  PLAYER_URL: ${PLAYER_URL}
+  DB_PASSWORD_FILE: ${DB_PASSWORD_FILE}
+  DB_HOST: ${DB_HOST}
+  DB_USER: ${DB_USER}
+  DB_NAME: ${DB_NAME}
+  JWT_SECRET_FILE: ${JWT_SECRET_FILE}
+  JWT_EXPIRY: ${JWT_EXPIRY}
+  CORS_ORIGIN: ${CORS_ORIGIN}
+  CORS_CREDENTIALS: ${CORS_CREDENTIALS}
+  CORS_PROXY_URL: ${CORS_PROXY_URL}
+  LIVEPEER_API_KEY_FILE: ${LIVEPEER_API_KEY_FILE}
+  THIRDWEB_SECRET_KEY_FILE: ${THIRDWEB_SECRET_KEY_FILE}
+  SPACES_KEY_FILE: ${SPACES_KEY_FILE}
+  SPACES_SECRET_FILE: ${SPACES_SECRET_FILE}
+  BUCKET_NAME: ${BUCKET_NAME}
+  BUCKET_URL: ${BUCKET_URL}
+  LIVEPEER_WEBHOOK_SECRET_FILE: ${LIVEPEER_WEBHOOK_SECRET_FILE}
+  LIVEPEER_BASE_URL: ${LIVEPEER_BASE_URL}
+  TELEGRAM_API_KEY_FILE: ${TELEGRAM_API_KEY_FILE}
+  TELEGRAM_CHAT_ID_FILE: ${TELEGRAM_CHAT_ID_FILE}
+  OAUTH_SECRET_FILE: ${OAUTH_SECRET_FILE}
+  TWITTER_OAUTH_SECRET_FILE: ${TWITTER_OAUTH_SECRET_FILE}
+  TWITTER_CLIENT_ID_FILE: ${TWITTER_CLIENT_ID_FILE}
+  GOOGLE_OAUTH_SECRET_FILE: ${GOOGLE_OAUTH_SECRET_FILE}
+  GOOGLE_CLIENT_ID_FILE: ${GOOGLE_CLIENT_ID_FILE}
+  SERVICE_ACCOUNT_PRIVATE_KEY_FILE: ${SERVICE_ACCOUNT_PRIVATE_KEY_FILE}
+  SERVICE_ACCOUNT_EMAIL_FILE: ${SERVICE_ACCOUNT_EMAIL_FILE}
+  MAIL_HOST: ${MAIL_HOST}
+  MAIL_PORT: ${MAIL_PORT}
+  MAIL_USER_FILE: ${MAIL_USER_FILE}
+  MAIL_PASS_FILE: ${MAIL_PASS_FILE}
+  MAGIC_LINK_SECRET_FILE: ${MAGIC_LINK_SECRET_FILE}
+  MAGIC_LINK_EXPIRY: ${MAGIC_LINK_EXPIRY}
+  WALLET_ADDRESSES: ${WALLET_ADDRESSES}
+  REMOTION_BASE_URL: http://reel-creator:4000
+  REMOTION_WEBHOOK_SECRET_FILE: ${LIVEPEER_WEBHOOK_SECRET_FILE}
+  REMOTION_ID: pebs
+  PIPEDREAM_AUTH_TOKEN_FILE: ${PIPEDREAM_AUTH_TOKEN_FILE}
+  DEVCON_UPLOAD_ENDPOINT_FILE: ${DEVCON_UPLOAD_ENDPOINT_FILE}
+  LOG_DIR: ${LOG_DIR}
+  LOG_FORMAT: ${LOG_FORMAT}
+  REDIS_PASSWORD_FILE: ${REDIS_PASSWORD_FILE}
+  REDIS_HOST: redis
+  REDIS_PORT: 6379
+  OPENAI_API_KEY_FILE: ${OPENAI_API_KEY_FILE}
+
+x-common-worker: &common-worker
+  volumes:
+    - ./packages/server:/app/packages/server
+    - /app/packages/server/node_modules
+  depends_on:
+    - mongodb
+    - redis
+
+services:
+  session-transcriptions:
+    <<: *common-worker
+    build:
+      context: .
+      dockerfile: ./packages/server/workers/session-transcriptions/Dockerfile.dev
+    environment: *common-environment
+
+  clips:
+    <<: *common-worker
+    build:
+      context: .
+      dockerfile: packages/server/workers/clips/Dockerfile.dev
+    environment: 
+      <<: *common-environment
+      REMOTION_WEBHOOK_URL: ${REMOTION_WEBHOOK_URL}
+
+  reel-creator:
+    build:
+      context: .
+      dockerfile: packages/reel-creator/Dockerfile.dev
+    volumes:
+      - ./packages/reel-creator:/app/packages/reel-creator
+      - /app/packages/reel-creator/node_modules
+    environment:
+      NODE_ENV: ${NODE_ENV}
+      SERVER_WEBHOOK_URL: https://kodiak-feasible-distinctly.ngrok-free.app/webhook/remotion
+      SERVER_WEBHOOK_SECRET_FILE: ${LIVEPEER_WEBHOOK_SECRET_FILE}
+      AWS_ACCESS_KEY_ID_FILE: ${AWS_ACCESS_KEY_ID_FILE}
+      AWS_SECRET_ACCESS_KEY_FILE: ${AWS_SECRET_ACCESS_KEY_FILE}
+      SITE_NAME: local-dev
+    ports:
+      - "4000:4000"
+
+  stage-transcriptions:
+    <<: *common-worker
+    build:
+      context: .
+      dockerfile: packages/server/workers/stage-transcriptions/Dockerfile.dev
+    environment:
+      <<: *common-environment
+
+  server:
+    <<: *common-worker
+    build: 
+      context: .
+      dockerfile: packages/server/src/Dockerfile.dev
+    ports:
+      - "3400:3400"
+    environment:
+      <<: *common-environment
+
+  mongodb:
+    image: mongo:latest
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongodb_data:/data/db
+      - ./mongo-init:/docker-entrypoint-initdb.d
+    environment:
+      MONGO_INITDB_DATABASE: streameth
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: ${DB_PASSWORD_FILE}
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongosh redis:27017/test --quiet
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
+    command: --auth --bind_ip_all
+
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379:6379"
+    command: redis-server --requirepass ${REDIS_PASSWORD_FILE}
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: sh -c "redis-cli -a $$(cat ${REDIS_PASSWORD_FILE}) ping"
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  mongodb_data:
+  redis_data: