Merge pull request #4 from Klimatbyran/feat/k8s

Feat/k8s (WIP)

.github/workflows/ci.yml

@@ -0,0 +1,85 @@
+name: Continuous Integration
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+on:
+  push:
+    branches:
+      - main
+      - feat/k8s
+    tags:
+      - production*
+      - staging*
+    paths-ignore:
+      - 'k8s/**'
+
+jobs:
+  version:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v3
+
+      - name: 🎫 Update patch version
+        run: |
+          git fetch
+          git checkout main
+          git pull origin main
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          if [[ $GITHUB_REF == refs/tags/production* ]]; then
+            npm version major
+          elif [[ $GITHUB_REF == refs/tags/staging* ]]; then
+            npm version minor
+          else
+            npm version patch
+          fi
+          git fetch --all
+          git push origin main
+
+  ci:
+    runs-on: ubuntu-latest
+    needs: version
+
+    steps:
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v3
+
+      - id: imagename
+        uses: ASzc/change-string-case-action@v2
+        with:
+          string: ${{ github.repository }}
+
+      - run: |
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          if [[ $GITHUB_REF == refs/tags/production* ]]; then
+            npm version major
+          elif [[ $GITHUB_REF == refs/tags/staging* ]]; then
+            npm version minor
+          else
+            npm version patch
+          fi
+
+      - name: 📝 Get Current Version
+        id: package-version
+        uses: martinbeentjes/npm-get-version-action@main
+
+      - name: 🔐 Login to Docker Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🔧 Build and push Docker Image
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ steps.imagename.outputs.lowercase  }}:${{ steps.package-version.outputs.current-version}}

.github/workflows/production.yml

@@ -0,0 +1,33 @@
+name: Production
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  production:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v3
+
+      - name: 🔐 Login to Docker Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🔧 Build and push Docker Image
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/Klimatbyran/data-pipeline:release-${{ github.event.release.tag_name }}

Dockerfile

@@ -0,0 +1,7 @@
+FROM node
+COPY package*.json /app/
+WORKDIR /app
+RUN npm install --omit=dev
+
+COPY . /app
+CMD ["npm", "start"]

k8s/deployment.yaml

@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: data-pipeline
+  namespace: data-pipeline
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: data-pipeline
+  template:
+    metadata:
+      labels:
+        app: data-pipeline
+    spec:
+      containers:
+        - image: ghcr.io/klimatbyran/data-pipeline # {"$imagepolicy": "flux-system:data-pipeline"}
+          resources:
+            limits:
+              cpu: 500m
+              memory: 256Mi
+            requests:
+              cpu: 100m
+              memory: 128Mi
+          name: data-pipeline
+          ports:
+            - containerPort: 80
+          env:
+            - name: DEBUG
+              value: 'true'
+            - name: CHROMA__PATH
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: CHROMA__PATH
+            - name: CHROMA__COLLECTION
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: CHROMA__COLLECTION
+            - name: CHROMA__TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: CHROMA__TOKEN
+            - name: OPENAI__API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: OPENAI__API_KEY
+            - name: MISTRAL__API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: MISTRAL__API_KEY
+            - name: MISTRAL__BASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: MISTRAL__BASE_URL
+            - name: SLACK__BOT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: SLACK__BOT_TOKEN
+            - name: SLACK__SIGNING_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: SLACK__SIGNING_SECRET
+            - name: SLACK__CHANNEL_IDS
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: SLACK__CHANNEL_IDS
+            - name: SLACK__APP_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: SLACK__APP_TOKEN
+            - name: OUTLINE__TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: OUTLINE__TOKEN
+            - name: OUTLINE__API_URL
+              valueFrom:
+                secretKeyRef:
+                  name: env
+                  key: OUTLINE__API_URL
+          imagePullPolicy: Always
+      imagePullSecrets:
+        - name: regcred

k8s/ingress.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: data-pipeline
+  annotations:
+    kubernetes.io/ingress.class: 'nginx'
+    cert-manager.io/cluster-issuer: 'letsencrypt-prod'
+    external-dns.alpha.kubernetes.io/hostname: pipeline.klimatkollen.se.
+spec:
+  tls:
+    - hosts:
+        - pipeline.klimatkollen.se
+      secretName: pipeline-klimatkollen-se-tls
+
+  rules:
+    - host: pipeline.klimatkollen.se
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: data-pipeline
+                port:
+                  number: 80

k8s/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: data-pipeline

k8s/service.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: data-pipeline
+spec:
+  ports:
+    - port: 80
+  selector:
+    app: data-pipeline

src/queues/index.ts

@@ -1,7 +1,7 @@
-import { Queue } from 'bullmq'
+import { Queue, QueueOptions } from 'bullmq'
 import redis from '../config/redis'
 
-const options = {
+const options: QueueOptions = {
   connection: redis,
   defaultJobOptions: { removeOnComplete: false },
 }

src/workers/searchVectors.ts

@@ -38,10 +38,16 @@ const worker = new Worker(
 
     job.log(JSON.stringify(results))
 
-    parseText.add('parse ' + url, {
-      url,
-      paragraphs: results.documents.flat(),
-    })
+    parseText.add(
+      'parse ' + url,
+      {
+        url,
+        paragraphs: results.documents.flat(),
+      },
+      {
+        attempts: 5,
+      }
+    )
 
     return results.documents
   },