diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 5256b32a..00000000 --- a/.dockerignore +++ /dev/null @@ -1,16 +0,0 @@ -# flyctl launch added from .gitignore -**/.DS_Store -**/node_modules -**/coverage -**/.tmp -**/tmp -**/proto/build -dist -!drizzle/**/*.sql -**/.eslintcache -**/docs/api/html/* -**/test/fixtures/config/*.zip - -# flyctl launch added from .husky/_/.gitignore -.husky/_/**/* -fly.toml diff --git a/.github/workflows/fly-cleanup.yml b/.github/workflows/fly-cleanup.yml deleted file mode 100644 index aeee93d1..00000000 --- a/.github/workflows/fly-cleanup.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Cleans up orphaned test apps on Fly - -# TODO: check app creation date - could destroy an app during a test run -# See . - -name: Fly Cleanup -on: - workflow_dispatch: - schedule: - - cron: '0 5 * * *' # Every day at 5am UTC -jobs: - cleanup: - name: Cleanup Orphaned Apps - runs-on: ubuntu-latest - concurrency: deploy-group # optional: ensure only one action runs at a time - steps: - - uses: actions/checkout@v4 - - uses: superfly/flyctl-actions/setup-flyctl@master - - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - run: | - fly apps list -q -o digidem | while IFS= read -r name; do - # Trim leading and trailing whitespace from $name - name=$(echo "$name" | xargs) - # Check if the name starts with 'comapeo-cloud-test-' - if [[ $name == comapeo-cloud-test-* ]]; then - # Call the fly destroy command with the name - fly apps destroy -y "$name" - fi - done diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml deleted file mode 100644 index bde16444..00000000 --- a/.github/workflows/server-test.yml +++ /dev/null @@ -1,26 +0,0 @@ -# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions - -name: Server e2e cloud test - -on: - push: - branches: [main] - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20.x - - uses: superfly/flyctl-actions/setup-flyctl@master - - run: npm ci - - run: npm run build --if-present - - run: node --test ./test-e2e/server.js - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - REMOTE_TEST_SERVER: true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 9074b8b4..00000000 --- a/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# Best practices from https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/ - -ARG NODE_VERSION=20.17.0 - -# --------------> The build image__ -FROM node:${NODE_VERSION} AS build -RUN apt-get update && apt-get install -y --no-install-recommends dumb-init -WORKDIR /usr/src/app -COPY package*.json /usr/src/app/ -RUN npm ci --omit=dev - -# --------------> The production image__ -FROM node:${NODE_VERSION}-bullseye-slim - -ENV NODE_ENV production -ENV PORT 8080 -EXPOSE 8080 -COPY --from=build /usr/bin/dumb-init /usr/bin/dumb-init -USER node -WORKDIR /usr/src/app -COPY --chown=node:node --from=build /usr/src/app/node_modules /usr/src/app/node_modules -COPY --chown=node:node . /usr/src/app -CMD ["dumb-init", "node", "src/server/server.js"] diff --git a/fly.toml b/fly.toml deleted file mode 100644 index a2e04897..00000000 --- a/fly.toml +++ /dev/null @@ -1,36 +0,0 @@ -# fly.toml app configuration file generated for comapeo-cloud on 2024-10-07T20:59:21+01:00 -# -# See https://fly.io/docs/reference/configuration/ for information about how to use this file. -# - -app = 'comapeo-cloud' -primary_region = 'iad' - -[env] - STORAGE_DIR = '/data' - -[build] - -[http_service] - internal_port = 8080 - force_https = true - auto_stop_machines = 'suspend' - auto_start_machines = true - min_machines_running = 0 - max_machines_running = 1 - processes = ['app'] - -[[http_service.checks]] - grace_period = "10s" - interval = "30s" - method = "GET" - timeout = "5s" - path = "/healthcheck" - -[[vm]] - size = 'shared-cpu-1x' - -[mounts] - source = "myapp_data" - destination = "/data" - snapshot_retention = 14 diff --git a/package-lock.json b/package-lock.json index 9f080c15..37ef0fc3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,9 +13,7 @@ "@comapeo/schema": "1.2.0", "@digidem/types": "^2.3.0", "@fastify/error": "^3.4.1", - "@fastify/sensible": "^5.6.0", "@fastify/type-provider-typebox": "^4.1.0", - "@fastify/websocket": "^10.0.1", "@hyperswarm/secret-stream": "^6.6.3", "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", @@ -30,7 +28,6 @@ "debug": "^4.3.4", "dot-prop": "^9.0.0", "drizzle-orm": "^0.30.8", - "env-schema": "^6.0.0", "fastify": "^4.0.0", "fastify-plugin": "^4.5.1", "hyperblobs": "2.3.0", @@ -65,8 +62,6 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1", - "@fastify/ajv-compiler": "^4.0.1", - "@fastify/fast-json-stringify-compiler": "^5.0.1", "@mapeo/default-config": "5.0.0", "@mapeo/mock-data": "^2.1.1", "@sinonjs/fake-timers": "^10.0.2", @@ -91,7 +86,6 @@ "cpy-cli": "^5.0.0", "drizzle-kit": "^0.20.14", "eslint": "^8.57.0", - "execa": "^9.4.0", "husky": "^8.0.0", "iterpal": "^0.4.0", "lint-staged": "^14.0.1", @@ -527,60 +521,6 @@ "node": ">=14" } }, - "node_modules/@fastify/ajv-compiler": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.1.tgz", - "integrity": "sha512-DxrBdgsjNLP0YM6W5Hd6/Fmj43S8zMKiFJYgi+Ri3htTGAowPVG/tG1wpnWLMjufEnehRivUCKZ1pLDIoZdTuw==", - "dev": true, - "dependencies": { - "ajv": "^8.12.0", - "ajv-formats": "^3.0.1", - "fast-uri": "^3.0.0" - } - }, - "node_modules/@fastify/ajv-compiler/node_modules/ajv": { - "version": "8.12.0", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@fastify/ajv-compiler/node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "dev": true, - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/@fastify/ajv-compiler/node_modules/fast-uri": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.2.tgz", - "integrity": "sha512-GR6f0hD7XXyNJa25Tb9BuIdN0tdr+0BMi6/CJPH3wJO1JjNG3n/VsSw38AwRdKZABm8lGbPfakLRkYzx2V9row==", - "dev": true - }, - "node_modules/@fastify/ajv-compiler/node_modules/json-schema-traverse": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, "node_modules/@fastify/deepmerge": { "version": "1.3.0", "license": "MIT" @@ -590,90 +530,6 @@ "resolved": "https://registry.npmjs.org/@fastify/error/-/error-3.4.1.tgz", "integrity": "sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ==" }, - "node_modules/@fastify/fast-json-stringify-compiler": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-5.0.1.tgz", - "integrity": "sha512-f2d3JExJgFE3UbdFcpPwqNUEoHWmt8pAKf8f+9YuLESdefA0WgqxeT6DrGL4Yrf/9ihXNSKOqpjEmurV405meA==", - "dev": true, - "dependencies": { - "fast-json-stringify": "^6.0.0" - } - }, - "node_modules/@fastify/fast-json-stringify-compiler/node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@fastify/fast-json-stringify-compiler/node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "dev": true, - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/@fastify/fast-json-stringify-compiler/node_modules/ajv/node_modules/fast-uri": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.2.tgz", - "integrity": "sha512-GR6f0hD7XXyNJa25Tb9BuIdN0tdr+0BMi6/CJPH3wJO1JjNG3n/VsSw38AwRdKZABm8lGbPfakLRkYzx2V9row==", - "dev": true - }, - "node_modules/@fastify/fast-json-stringify-compiler/node_modules/fast-json-stringify": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-6.0.0.tgz", - "integrity": "sha512-FGMKZwniMTgZh7zQp9b6XnBVxUmKVahQLQeRQHqwYmPDqDhcEKZ3BaQsxelFFI5PY7nN71OEeiL47/zUWcYe1A==", - "dev": true, - "dependencies": { - "@fastify/merge-json-schemas": "^0.1.1", - "ajv": "^8.12.0", - "ajv-formats": "^3.0.1", - "fast-deep-equal": "^3.1.3", - "fast-uri": "^2.3.0", - "json-schema-ref-resolver": "^1.0.1", - "rfdc": "^1.2.0" - } - }, - "node_modules/@fastify/fast-json-stringify-compiler/node_modules/fast-uri": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-2.4.0.tgz", - "integrity": "sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA==", - "dev": true - }, - "node_modules/@fastify/fast-json-stringify-compiler/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, - "node_modules/@fastify/merge-json-schemas": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@fastify/merge-json-schemas/-/merge-json-schemas-0.1.1.tgz", - "integrity": "sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.3" - } - }, "node_modules/@fastify/send": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@fastify/send/-/send-2.1.0.tgz", @@ -697,20 +553,6 @@ "node": ">=10.0.0" } }, - "node_modules/@fastify/sensible": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@fastify/sensible/-/sensible-5.6.0.tgz", - "integrity": "sha512-Vq6Z2ZQy10GDqON+hvLF52K99s9et5gVVxTul5n3SIAf0Kq5QjPRUKkAMT3zPAiiGvoHtS3APa/3uaxfDgCODQ==", - "dependencies": { - "@lukeed/ms": "^2.0.1", - "fast-deep-equal": "^3.1.1", - "fastify-plugin": "^4.0.0", - "forwarded": "^0.2.0", - "http-errors": "^2.0.0", - "type-is": "^1.6.18", - "vary": "^1.1.2" - } - }, "node_modules/@fastify/static": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/@fastify/static/-/static-7.0.4.tgz", @@ -775,16 +617,6 @@ "@sinclair/typebox": ">=0.26 <=0.33" } }, - "node_modules/@fastify/websocket": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/@fastify/websocket/-/websocket-10.0.1.tgz", - "integrity": "sha512-8/pQIxTPRD8U94aILTeJ+2O3el/r19+Ej5z1O1mXlqplsUH7KzCjAI0sgd5DM/NoPjAi5qLFNIjgM5+9/rGSNw==", - "dependencies": { - "duplexify": "^4.1.2", - "fastify-plugin": "^4.0.0", - "ws": "^8.0.0" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -1172,12 +1004,6 @@ "version": "1.1.0", "license": "BSD-3-Clause" }, - "node_modules/@sec-ant/readable-stream": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", - "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", - "dev": true - }, "node_modules/@shikijs/core": { "version": "1.17.7", "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-1.17.7.tgz", @@ -1234,18 +1060,6 @@ "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==" }, - "node_modules/@sindresorhus/merge-streams": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", - "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", - "dev": true, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@sinonjs/commons": { "version": "2.0.0", "dev": true, @@ -3164,25 +2978,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/dotenv": { - "version": "16.4.5", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", - "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dotenv-expand": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", - "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", - "engines": { - "node": ">=12" - } - }, "node_modules/dprint-node": { "version": "1.0.7", "dev": true, @@ -3405,17 +3200,6 @@ } } }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "license": "MIT" @@ -3455,41 +3239,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/env-schema": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/env-schema/-/env-schema-6.0.0.tgz", - "integrity": "sha512-/IHp1EmrfubUOfF1wfe8koDWM5/dxUDylHANPNrPyrsYWJ7KRiB8gXbjtqQBujmOhpSpXXOhhnaL+meb+MaGtA==", - "dependencies": { - "ajv": "^8.12.0", - "dotenv": "^16.4.5", - "dotenv-expand": "10.0.0" - } - }, - "node_modules/env-schema/node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/env-schema/node_modules/fast-uri": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.2.tgz", - "integrity": "sha512-GR6f0hD7XXyNJa25Tb9BuIdN0tdr+0BMi6/CJPH3wJO1JjNG3n/VsSw38AwRdKZABm8lGbPfakLRkYzx2V9row==" - }, - "node_modules/env-schema/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/error-ex": { "version": "1.3.2", "dev": true, @@ -3890,68 +3639,6 @@ "node": ">=0.8.x" } }, - "node_modules/execa": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-9.4.0.tgz", - "integrity": "sha512-yKHlle2YGxZE842MERVIplWwNH5VYmqqcPFgtnlU//K8gxuFFXu0pwd/CrfXTumFpeEiufsP7+opT/bPJa1yVw==", - "dev": true, - "dependencies": { - "@sindresorhus/merge-streams": "^4.0.0", - "cross-spawn": "^7.0.3", - "figures": "^6.1.0", - "get-stream": "^9.0.0", - "human-signals": "^8.0.0", - "is-plain-obj": "^4.1.0", - "is-stream": "^4.0.1", - "npm-run-path": "^6.0.0", - "pretty-ms": "^9.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^4.0.0", - "yoctocolors": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.5.0" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/execa/node_modules/is-plain-obj": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", - "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/execa/node_modules/is-stream": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", - "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", - "dev": true, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/execa/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/expand-template": { "version": "2.0.3", "license": "(MIT OR WTFPL)", @@ -4198,21 +3885,6 @@ "reusify": "^1.0.4" } }, - "node_modules/figures": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", - "integrity": "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==", - "dev": true, - "dependencies": { - "is-unicode-supported": "^2.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -4460,34 +4132,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-stream": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", - "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", - "dev": true, - "dependencies": { - "@sec-ant/readable-stream": "^0.4.1", - "is-stream": "^4.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-stream/node_modules/is-stream": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", - "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", - "dev": true, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-symbol-description": { "version": "1.0.0", "dev": true, @@ -4838,15 +4482,6 @@ "node": ">= 0.8" } }, - "node_modules/human-signals": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-8.0.0.tgz", - "integrity": "sha512-/1/GPCpDUCCYwlERiYjxoczfP0zfvZMU/OWgQPMya9AbAE24vseigFdhAMObpc8Q4lc/kjutPfUddDYyAmejnA==", - "dev": true, - "engines": { - "node": ">=18.18.0" - } - }, "node_modules/husky": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", @@ -5573,15 +5208,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/json-schema-ref-resolver": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-1.0.1.tgz", - "integrity": "sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.3" - } - }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -6276,14 +5902,6 @@ "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", "dev": true }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/memoizee": { "version": "0.4.15", "dev": true, @@ -6480,25 +6098,6 @@ "node": ">=16" } }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", @@ -6960,34 +6559,6 @@ "which": "bin/which" } }, - "node_modules/npm-run-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-6.0.0.tgz", - "integrity": "sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==", - "dev": true, - "dependencies": { - "path-key": "^4.0.0", - "unicorn-magic": "^0.3.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/npm-run-path/node_modules/path-key": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", - "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/object-inspect": { "version": "1.12.3", "dev": true, @@ -8950,11 +8521,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" - }, "node_modules/streamx": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.19.0.tgz", @@ -9118,18 +8684,6 @@ "node": ">=4" } }, - "node_modules/strip-final-newline": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-4.0.0.tgz", - "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", - "dev": true, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/strip-indent": { "version": "4.0.0", "dev": true, @@ -9573,18 +9127,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/typed-array-buffer": { "version": "1.0.0", "dev": true, @@ -9772,18 +9314,6 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, - "node_modules/unicorn-magic": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", - "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", - "dev": true, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", @@ -9909,14 +9439,6 @@ "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==" }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", diff --git a/package.json b/package.json index 8105f867..fee337a0 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "db:generate:project": "drizzle-kit generate:sqlite --schema src/schema/project.js --out drizzle/project", "db:generate:client": "drizzle-kit generate:sqlite --schema src/schema/client.js --out drizzle/client", "prepack": "npm run build:types", - "prepare": "husky install || true" + "prepare": "husky install" }, "files": [ "src", @@ -109,8 +109,6 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1", - "@fastify/ajv-compiler": "^4.0.1", - "@fastify/fast-json-stringify-compiler": "^5.0.1", "@mapeo/default-config": "5.0.0", "@mapeo/mock-data": "^2.1.1", "@sinonjs/fake-timers": "^10.0.2", @@ -135,7 +133,6 @@ "cpy-cli": "^5.0.0", "drizzle-kit": "^0.20.14", "eslint": "^8.57.0", - "execa": "^9.4.0", "husky": "^8.0.0", "iterpal": "^0.4.0", "lint-staged": "^14.0.1", @@ -161,9 +158,7 @@ "@comapeo/schema": "1.2.0", "@digidem/types": "^2.3.0", "@fastify/error": "^3.4.1", - "@fastify/sensible": "^5.6.0", "@fastify/type-provider-typebox": "^4.1.0", - "@fastify/websocket": "^10.0.1", "@hyperswarm/secret-stream": "^6.6.3", "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", @@ -178,7 +173,6 @@ "debug": "^4.3.4", "dot-prop": "^9.0.0", "drizzle-orm": "^0.30.8", - "env-schema": "^6.0.0", "fastify": "^4.0.0", "fastify-plugin": "^4.5.1", "hyperblobs": "2.3.0", diff --git a/src/server/README.md b/src/server/README.md deleted file mode 100644 index 8ce9582c..00000000 --- a/src/server/README.md +++ /dev/null @@ -1,38 +0,0 @@ -## Deploying CoMapeo Cloud - -CoMapeo Cloud comes with a [`Dockerfile`](../../Dockerfile) that can be used to build a Docker image. This image can be used to deploy CoMapeo Cloud on a server. - -Server configuration is done using environment variables. The following environment variables are available: - -| Environment Variable | Required | Description | Default Value | -| --------------------- | -------- | -------------------------------------------------------------------- | ---------------- | -| `SERVER_BEARER_TOKEN` | Yes | Token for authenticating API requests. Should be large random string | | -| `PORT` | No | Port on which the server runs | `8080` | -| `SERVER_NAME` | No | Friendly server name, seen by users when adding server | `CoMapeo Server` | -| `ALLOWED_PROJECTS` | No | Number of projects allowed to register with the server | `1` | -| `STORAGE_DIR` | No | Path for storing app & project data | `$CWD/data` | - -### Deploying with fly.io - -CoMapeo Cloud can be deployed on [fly.io](https://fly.io) using the following steps: - -1. Install the flyctl CLI tool by following the instructions [here](https://fly.io/docs/getting-started/installing-flyctl/). -2. Create a new app on fly.io by running `flyctl apps create`, take a note of the app name. -3. Set the SERVER_BEARER_TOKEN secret via: - ```sh - flyctl secrets set SERVER_BEARER_TOKEN= --app - ``` -4. Deploy the app by running (optionally setting the `ALLOWED_PROJECTS` environment variable): - ```sh - flyctl deploy --app -e ALLOWED_PROJECTS=10 - ``` -5. The app should now be running on fly.io. You can access it at `https://.fly.dev`. - -To destroy the app (delete all data and project invites), run: - -> [!WARNING] -> This action is irreversible and will permanently delete all data associated with the app, and projects that have already added the server will no longer be able to sync with it. - -```sh -flyctl destroy --app -``` diff --git a/src/server/allowed-hosts-plugin.js b/src/server/allowed-hosts-plugin.js deleted file mode 100644 index 732f05be..00000000 --- a/src/server/allowed-hosts-plugin.js +++ /dev/null @@ -1,19 +0,0 @@ -import createFastifyPlugin from 'fastify-plugin' - -/** - * @typedef {object} AllowedHostsPluginOptions - * @property {string[]} [allowedHosts] - */ - -/** @type {import('fastify').FastifyPluginAsync} */ -const comapeoPlugin = async function (fastify, { allowedHosts }) { - if (!allowedHosts) { - return - } - const allowedHostsSet = new Set(allowedHosts) - fastify.addHook('onRequest', async function (req) { - this.assert(allowedHostsSet.has(req.hostname), 403, 'Forbidden') - }) -} - -export default createFastifyPlugin(comapeoPlugin, { name: 'allowedHosts' }) diff --git a/src/server/app.js b/src/server/app.js deleted file mode 100644 index a87b292c..00000000 --- a/src/server/app.js +++ /dev/null @@ -1,46 +0,0 @@ -import fastifyWebsocket from '@fastify/websocket' -import fastifySensible from '@fastify/sensible' -import createFastifyPlugin from 'fastify-plugin' -import routes from './routes.js' -import comapeoPlugin from './comapeo-plugin.js' -import baseUrlPlugin from './base-url-plugin.js' -import allowedHostsPlugin from './allowed-hosts-plugin.js' -/** @import { FastifyServerOptions } from 'fastify' */ -/** @import { ComapeoPluginOptions } from './comapeo-plugin.js' */ -/** @import { RouteOptions } from './routes.js' */ - -/** - * @internal - * @typedef {object} OtherServerOptions - * @prop {string[]} [allowedHosts] - */ - -/** @typedef {ComapeoPluginOptions & OtherServerOptions & RouteOptions} ServerOptions */ - -/** @type {import('fastify').FastifyPluginAsync} */ -async function comapeoServer( - fastify, - { - serverBearerToken, - serverName, - allowedHosts, - allowedProjects = 1, - ...comapeoPluginOpts - } -) { - fastify.register(fastifyWebsocket) - fastify.register(fastifySensible, { sharedSchemaId: 'HttpError' }) - fastify.register(allowedHostsPlugin, { allowedHosts }) - fastify.register(baseUrlPlugin) - fastify.register(comapeoPlugin, comapeoPluginOpts) - fastify.register(routes, { - serverBearerToken, - serverName, - allowedProjects, - }) -} - -export default createFastifyPlugin(comapeoServer, { - name: 'comapeoServer', - fastify: '4.x', -}) diff --git a/src/server/base-url-plugin.js b/src/server/base-url-plugin.js deleted file mode 100644 index 81904978..00000000 --- a/src/server/base-url-plugin.js +++ /dev/null @@ -1,11 +0,0 @@ -import createFastifyPlugin from 'fastify-plugin' - -/** @type {import('fastify').FastifyPluginAsync} */ -const baseUrlPlugin = async function (fastify) { - fastify.decorateRequest('baseUrl', null) - fastify.addHook('onRequest', async function (req) { - req.baseUrl = new URL(this.prefix, `${req.protocol}://${req.hostname}`) - }) -} - -export default createFastifyPlugin(baseUrlPlugin, { name: 'baseUrl' }) diff --git a/src/server/comapeo-plugin.js b/src/server/comapeo-plugin.js deleted file mode 100644 index 03daa0fc..00000000 --- a/src/server/comapeo-plugin.js +++ /dev/null @@ -1,14 +0,0 @@ -import { MapeoManager } from '../index.js' -import createFastifyPlugin from 'fastify-plugin' - -/** - * @typedef {Omit[0], 'fastify'>} ComapeoPluginOptions - */ - -/** @type {import('fastify').FastifyPluginAsync} */ -const comapeoPlugin = async function (fastify, opts) { - const comapeo = new MapeoManager({ ...opts, fastify }) - fastify.decorate('comapeo', comapeo) -} - -export default createFastifyPlugin(comapeoPlugin, { name: 'comapeo' }) diff --git a/src/server/routes.js b/src/server/routes.js deleted file mode 100644 index 2ccdfaab..00000000 --- a/src/server/routes.js +++ /dev/null @@ -1,405 +0,0 @@ -import { keyToPublicId as projectKeyToPublicId } from '@mapeo/crypto' -import { Type } from '@sinclair/typebox' -import assert from 'node:assert/strict' -import * as fs from 'node:fs' -import timingSafeEqual from 'string-timing-safe-equal' -import { replicateProject } from '../index.js' -import { wsCoreReplicator } from './ws-core-replicator.js' -/** @import {FastifyInstance, FastifyPluginAsync, FastifyRequest, RawServerDefault} from 'fastify' */ -/** @import {TypeBoxTypeProvider} from '@fastify/type-provider-typebox' */ - -const BEARER_SPACE_LENGTH = 'Bearer '.length -const HEX_REGEX_32_BYTES = '^[0-9a-fA-F]{64}$' -const HEX_STRING_32_BYTES = Type.String({ pattern: HEX_REGEX_32_BYTES }) -const BASE32_REGEX_32_BYTES = '^[0-9A-Za-z]{52}$' -const BASE32_STRING_32_BYTES = Type.String({ pattern: BASE32_REGEX_32_BYTES }) - -const INDEX_HTML_PATH = new URL('./static/index.html', import.meta.url) - -/** - * @typedef {object} RouteOptions - * @prop {string} serverBearerToken - * @prop {string} serverName - * @prop {string[] | number} [allowedProjects=1] - */ - -/** @type {FastifyPluginAsync} */ -export default async function routes( - fastify, - { serverBearerToken, serverName, allowedProjects = 1 } -) { - /** @type {Set | number} */ - const allowedProjectsSetOrNumber = Array.isArray(allowedProjects) - ? new Set(allowedProjects) - : allowedProjects - - /** - * @param {FastifyRequest} req - */ - const verifyBearerAuth = (req) => { - if (!isBearerTokenValid(req.headers.authorization, serverBearerToken)) { - throw fastify.httpErrors.forbidden('Invalid bearer token') - } - } - - fastify.get('/', (_req, reply) => { - const stream = fs.createReadStream(INDEX_HTML_PATH) - reply.header('Content-Type', 'text/html') - reply.send(stream) - }) - - fastify.get( - '/info', - { - schema: { - response: { - 200: Type.Object({ - data: Type.Object({ - deviceId: Type.String(), - name: Type.String(), - }), - }), - 500: { $ref: 'HttpError' }, - }, - }, - }, - async function () { - const { deviceId, name } = this.comapeo.getDeviceInfo() - return { - data: { deviceId, name: name || serverName }, - } - } - ) - - fastify.get( - '/sync/:projectPublicId', - { - schema: { - params: Type.Object({ - projectPublicId: BASE32_STRING_32_BYTES, - }), - response: { - 404: { $ref: 'HttpError' }, - }, - }, - async preHandler(req) { - await ensureProjectExists(this, req) - }, - websocket: true, - }, - async function (socket, req) { - // The preValidation hook ensures that the project exists - const project = await this.comapeo.getProject(req.params.projectPublicId) - const replicationStream = replicateProject(project, false) - wsCoreReplicator(socket, replicationStream) - project.$sync.start() - } - ) - - fastify.get( - '/projects', - { - schema: { - response: { - 200: Type.Object({ - data: Type.Array( - Type.Object({ - projectId: Type.String(), - name: Type.String(), - }) - ), - }), - 403: { $ref: 'HttpError' }, - }, - }, - async preHandler(req) { - verifyBearerAuth(req) - }, - }, - async function () { - const projects = await this.comapeo.listProjects() - return { - data: projects.map((project) => ({ - projectId: project.projectId, - name: project.name, - })), - } - } - ) - - fastify.put( - '/projects', - { - schema: { - body: Type.Object({ - projectName: Type.String({ minLength: 1 }), - projectKey: HEX_STRING_32_BYTES, - encryptionKeys: Type.Object({ - auth: HEX_STRING_32_BYTES, - config: HEX_STRING_32_BYTES, - data: HEX_STRING_32_BYTES, - blobIndex: HEX_STRING_32_BYTES, - blob: HEX_STRING_32_BYTES, - }), - }), - response: { - 200: Type.Object({ - data: Type.Object({ - deviceId: HEX_STRING_32_BYTES, - }), - }), - 400: { $ref: 'HttpError' }, - }, - }, - }, - async function (req) { - const { projectName } = req.body - const projectKey = Buffer.from(req.body.projectKey, 'hex') - const projectPublicId = projectKeyToPublicId(projectKey) - - const existingProjects = await this.comapeo.listProjects() - - // This assumes that two projects with the same project key are equivalent, - // and that we don't need to add more. Theoretically, someone could add - // project with ID 1 and keys A, then add project with ID 1 and keys B. - // This would mean a malicious/buggy client, which could cause errors if - // trying to sync with this server--that seems acceptable. - const alreadyHasThisProject = existingProjects.some((p) => - // We don't want people to be able to enumerate the project keys that - // this server has. - timingSafeEqual(p.projectId, projectPublicId) - ) - - if (!alreadyHasThisProject) { - if ( - allowedProjectsSetOrNumber instanceof Set && - !allowedProjectsSetOrNumber.has(projectPublicId) - ) { - throw fastify.httpErrors.forbidden('Project not allowed') - } - - if ( - typeof allowedProjectsSetOrNumber === 'number' && - existingProjects.length >= allowedProjectsSetOrNumber - ) { - throw fastify.httpErrors.forbidden( - 'Server is already linked to the maximum number of projects' - ) - } - } - - const baseUrl = req.baseUrl.toString() - - const existingDeviceInfo = this.comapeo.getDeviceInfo() - // We don't set device info until this point. We trust that `req.hostname` - // is the hostname we want clients to use to sync to the server. - if ( - existingDeviceInfo.deviceType === 'device_type_unspecified' || - existingDeviceInfo.selfHostedServerDetails?.baseUrl !== baseUrl - ) { - await this.comapeo.setDeviceInfo({ - deviceType: 'selfHostedServer', - name: serverName, - selfHostedServerDetails: { baseUrl }, - }) - } - - if (!alreadyHasThisProject) { - const projectId = await this.comapeo.addProject( - { - projectKey, - projectName, - encryptionKeys: { - auth: Buffer.from(req.body.encryptionKeys.auth, 'hex'), - config: Buffer.from(req.body.encryptionKeys.config, 'hex'), - data: Buffer.from(req.body.encryptionKeys.data, 'hex'), - blobIndex: Buffer.from(req.body.encryptionKeys.blobIndex, 'hex'), - blob: Buffer.from(req.body.encryptionKeys.blob, 'hex'), - }, - }, - { waitForSync: false } - ) - assert.equal( - projectId, - projectPublicId, - 'adding a project should return the same ID as what was passed' - ) - } - - const project = await this.comapeo.getProject(projectPublicId) - project.$sync.start() - - return { - data: { - deviceId: this.comapeo.deviceId, - }, - } - } - ) - - fastify.get( - '/projects/:projectPublicId/observations', - { - schema: { - params: Type.Object({ - projectPublicId: BASE32_STRING_32_BYTES, - }), - response: { - 200: Type.Object({ - data: Type.Array( - Type.Object({ - docId: Type.String(), - createdAt: Type.String(), - updatedAt: Type.String(), - deleted: Type.Boolean(), - lat: Type.Optional(Type.Number()), - lon: Type.Optional(Type.Number()), - attachments: Type.Array( - Type.Object({ - url: Type.String(), - }) - ), - tags: Type.Record( - Type.String(), - Type.Union([ - Type.Boolean(), - Type.Number(), - Type.String(), - Type.Null(), - Type.Array( - Type.Union([ - Type.Boolean(), - Type.Number(), - Type.String(), - Type.Null(), - ]) - ), - ]) - ), - }) - ), - }), - 403: { $ref: 'HttpError' }, - 404: { $ref: 'HttpError' }, - }, - }, - async preHandler(req) { - verifyBearerAuth(req) - await ensureProjectExists(this, req) - }, - }, - async function (req) { - const { projectPublicId } = req.params - const project = await this.comapeo.getProject(projectPublicId) - - return { - data: (await project.observation.getMany({ includeDeleted: true })).map( - (obs) => ({ - docId: obs.docId, - createdAt: obs.createdAt, - updatedAt: obs.updatedAt, - deleted: obs.deleted, - lat: obs.lat, - lon: obs.lon, - attachments: obs.attachments - // TODO: For now, only photos are supported. - // See . - .filter((attachment) => attachment.type === 'photo') - .map((attachment) => ({ - url: new URL( - `projects/${projectPublicId}/attachments/${attachment.driveDiscoveryId}/${attachment.type}/${attachment.name}`, - req.baseUrl - ).href, - })), - tags: obs.tags, - }) - ), - } - } - ) - - fastify.get( - '/projects/:projectPublicId/attachments/:driveDiscoveryId/:type/:name', - { - schema: { - params: Type.Object({ - projectPublicId: BASE32_STRING_32_BYTES, - driveDiscoveryId: Type.String(), - // TODO: For now, only photos are supported. - // See . - type: Type.Literal('photo'), - name: Type.String(), - }), - querystring: Type.Object({ - variant: Type.Optional( - Type.Union([ - Type.Literal('original'), - Type.Literal('preview'), - Type.Literal('thumbnail'), - ]) - ), - }), - response: { - 403: { $ref: 'HttpError' }, - 404: { $ref: 'HttpError' }, - }, - }, - async preHandler(req) { - verifyBearerAuth(req) - await ensureProjectExists(this, req) - }, - }, - async function (req, reply) { - const project = await this.comapeo.getProject(req.params.projectPublicId) - - const blobUrl = await project.$blobs.getUrl({ - driveId: req.params.driveDiscoveryId, - name: req.params.name, - type: req.params.type, - variant: req.query.variant || 'original', - }) - - const proxiedResponse = await fetch(blobUrl) - reply.code(proxiedResponse.status) - for (const [headerName, headerValue] of proxiedResponse.headers) { - reply.header(headerName, headerValue) - } - return reply.send(proxiedResponse.body) - } - ) -} - -/** - * @param {FastifyInstance} fastify - * @param {object} req - * @param {object} req.params - * @param {string} req.params.projectPublicId - * @returns {Promise} - */ -async function ensureProjectExists(fastify, req) { - try { - await fastify.comapeo.getProject(req.params.projectPublicId) - } catch (e) { - if (e instanceof Error && e.message.startsWith('NotFound')) { - throw fastify.httpErrors.notFound('Project not found') - } - throw e - } -} - -/** - * @param {undefined | string} headerValue - * @param {string} expectedBearerToken - * @returns {boolean} - */ -function isBearerTokenValid(headerValue = '', expectedBearerToken) { - // This check is not strictly required for correctness, but helps protect - // against long values. - const expectedLength = BEARER_SPACE_LENGTH + expectedBearerToken.length - if (headerValue.length !== expectedLength) return false - - if (!headerValue.startsWith('Bearer ')) return false - const actualBearerToken = headerValue.slice(BEARER_SPACE_LENGTH) - - return timingSafeEqual(actualBearerToken, expectedBearerToken) -} diff --git a/src/server/server.js b/src/server/server.js deleted file mode 100644 index b088b823..00000000 --- a/src/server/server.js +++ /dev/null @@ -1,106 +0,0 @@ -import { Type } from '@sinclair/typebox' -import envSchema from 'env-schema' -import createFastify from 'fastify' -import crypto from 'node:crypto' -import fsPromises from 'node:fs/promises' -import path from 'node:path' -import comapeoServer from './app.js' - -const DEFAULT_STORAGE = path.join(process.cwd(), 'data') -const CORE_DIR_NAME = 'core' -const DB_DIR_NAME = 'db' -const ROOT_KEY_FILE_NAME = 'root-key' - -const schema = Type.Object({ - PORT: Type.Number({ default: 8080 }), - SERVER_NAME: Type.String({ - description: 'name of the server', - default: 'CoMapeo Server', - }), - SERVER_BEARER_TOKEN: Type.String({ - description: - 'Bearer token for accessing the server, can be any random string', - }), - STORAGE_DIR: Type.String({ - description: 'path to directory where data is stored', - default: DEFAULT_STORAGE, - }), - ALLOWED_PROJECTS: Type.Optional( - Type.Integer({ - minimum: 1, - description: 'number of projects allowed to join the server', - }) - ), -}) - -/** @typedef {import('@sinclair/typebox').Static} Env */ -/** @type {ReturnType>} */ -const config = envSchema({ schema, dotenv: true }) - -const coreStorage = path.join(config.STORAGE_DIR, CORE_DIR_NAME) -const dbFolder = path.join(config.STORAGE_DIR, DB_DIR_NAME) -const rootKeyFile = path.join(config.STORAGE_DIR, ROOT_KEY_FILE_NAME) - -const migrationsFolder = new URL('../../drizzle/', import.meta.url).pathname -const projectMigrationsFolder = path.join(migrationsFolder, 'project') -const clientMigrationsFolder = path.join(migrationsFolder, 'client') - -await Promise.all([ - fsPromises.mkdir(coreStorage, { recursive: true }), - fsPromises.mkdir(dbFolder, { recursive: true }), -]) - -/** @type {Buffer} */ -let rootKey -try { - rootKey = await fsPromises.readFile(rootKeyFile) -} catch (err) { - if ( - typeof err === 'object' && - err && - 'code' in err && - err.code !== 'ENOENT' - ) { - throw err - } - rootKey = crypto.randomBytes(16) - await fsPromises.writeFile(rootKeyFile, rootKey) -} - -if (!rootKey || rootKey.length !== 16) { - throw new Error('Root key must be 16 bytes') -} - -const fastify = createFastify({ - logger: true, - trustProxy: true, -}) -fastify.register(comapeoServer, { - serverName: config.SERVER_NAME, - serverBearerToken: config.SERVER_BEARER_TOKEN, - allowedProjects: config.ALLOWED_PROJECTS, - rootKey, - coreStorage, - dbFolder, - projectMigrationsFolder, - clientMigrationsFolder, -}) - -fastify.get('/healthcheck', async () => {}) - -try { - await fastify.listen({ port: config.PORT, host: '0.0.0.0' }) -} catch (err) { - fastify.log.error(err) - process.exit(1) -} - -/** @param {NodeJS.Signals} signal*/ -async function closeGracefully(signal) { - console.log(`Received signal to terminate: ${signal}`) - await fastify.close() - console.log('Gracefully closed fastify') - process.kill(process.pid, signal) -} -process.once('SIGINT', closeGracefully) -process.once('SIGTERM', closeGracefully) diff --git a/src/server/static/index.html b/src/server/static/index.html deleted file mode 100644 index d7b253b1..00000000 --- a/src/server/static/index.html +++ /dev/null @@ -1,43 +0,0 @@ - - - - - CoMapeo - - - - -

¡Hola desde CoMapeo!

-

Olá da CoMapeo!

-

Hello from CoMapeo!

- - diff --git a/src/server/test/add-project-endpoint.js b/src/server/test/add-project-endpoint.js deleted file mode 100644 index 595bdee9..00000000 --- a/src/server/test/add-project-endpoint.js +++ /dev/null @@ -1,223 +0,0 @@ -import { keyToPublicId as projectKeyToPublicId } from '@mapeo/crypto' -import assert from 'node:assert/strict' -import test from 'node:test' -import { - createTestServer, - randomAddProjectBody, - randomHex, -} from './test-helpers.js' - -test('request missing project name', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: omit(randomAddProjectBody(), 'projectName'), - }) - - assert.equal(response.statusCode, 400) -}) - -test('request with empty project name', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: { ...randomAddProjectBody(), projectName: '' }, - }) - - assert.equal(response.statusCode, 400) -}) - -test('request missing project key', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: omit(randomAddProjectBody(), 'projectKey'), - }) - - assert.equal(response.statusCode, 400) -}) - -test("request with a project key that's too short", async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: { ...randomAddProjectBody(), projectKey: randomHex(31) }, - }) - - assert.equal(response.statusCode, 400) -}) - -test('request missing any encryption keys', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: omit(randomAddProjectBody(), 'encryptionKeys'), - }) - - assert.equal(response.statusCode, 400) -}) - -test('request missing an encryption key', async (t) => { - const server = createTestServer(t) - const body = randomAddProjectBody() - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: { - ...body, - encryptionKeys: omit(body.encryptionKeys, 'config'), - }, - }) - - assert.equal(response.statusCode, 400) -}) - -test("request with an encryption key that's too short", async (t) => { - const server = createTestServer(t) - const body = randomAddProjectBody() - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: { - ...body, - encryptionKeys: { ...body.encryptionKeys, config: randomHex(31) }, - }, - }) - - assert.equal(response.statusCode, 400) -}) - -test('adding a project', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: randomAddProjectBody(), - }) - - assert.equal(response.statusCode, 200) - assert.deepEqual(response.json(), { - data: { deviceId: server.deviceId }, - }) -}) - -test('adding a second project fails by default', async (t) => { - const server = createTestServer(t) - - const firstAddResponse = await server.inject({ - method: 'PUT', - url: '/projects', - body: randomAddProjectBody(), - }) - assert.equal(firstAddResponse.statusCode, 200) - - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: randomAddProjectBody(), - }) - assert.equal(response.statusCode, 403) - assert.match(response.json().message, /maximum number of projects/) -}) - -test('allowing a maximum number of projects', async (t) => { - const server = createTestServer(t, { allowedProjects: 3 }) - - await t.test('adding 3 projects', async () => { - for (let i = 0; i < 3; i++) { - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: randomAddProjectBody(), - }) - assert.equal(response.statusCode, 200) - } - }) - - await t.test('attempting to add 4th project fails', async () => { - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: randomAddProjectBody(), - }) - assert.equal(response.statusCode, 403) - assert.match(response.json().message, /maximum number of projects/) - }) -}) - -test( - 'allowing a specific list of projects', - { concurrency: true }, - async (t) => { - const body = randomAddProjectBody() - const projectPublicId = projectKeyToPublicId( - Buffer.from(body.projectKey, 'hex') - ) - const server = createTestServer(t, { - allowedProjects: [projectPublicId], - }) - - await t.test('adding a project in the list', async () => { - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body, - }) - assert.equal(response.statusCode, 200) - }) - - await t.test('trying to add a project not in the list', async () => { - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body: randomAddProjectBody(), - }) - assert.equal(response.statusCode, 403) - }) - } -) - -test('adding the same project twice is idempotent', async (t) => { - const server = createTestServer(t, { allowedProjects: 1 }) - const body = randomAddProjectBody() - - const firstResponse = await server.inject({ - method: 'PUT', - url: '/projects', - body, - }) - assert.equal(firstResponse.statusCode, 200) - - const secondResponse = await server.inject({ - method: 'PUT', - url: '/projects', - body, - }) - assert.equal(secondResponse.statusCode, 200) -}) - -/** - * @template {object} T - * @template {keyof T} K - * @param {T} obj - * @param {K} key - * @returns {Omit} - */ -function omit(obj, key) { - const result = { ...obj } - delete result[key] - return result -} diff --git a/src/server/test/allowed-hosts.js b/src/server/test/allowed-hosts.js deleted file mode 100644 index 4cab3b40..00000000 --- a/src/server/test/allowed-hosts.js +++ /dev/null @@ -1,28 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { createTestServer } from './test-helpers.js' - -test('allowed host', async (t) => { - const allowedHost = 'www.example.com' - const server = createTestServer(t, { allowedHosts: [allowedHost] }) - - const response = await server.inject({ - authority: allowedHost, - method: 'GET', - url: '/info', - }) - - assert.equal(response.statusCode, 200) -}) - -test('disallowed host', async (t) => { - const server = createTestServer(t, { allowedHosts: ['www.example.com'] }) - - const response = await server.inject({ - authority: 'www.invalid-host.example', - method: 'GET', - url: '/info', - }) - - assert.equal(response.statusCode, 403) -}) diff --git a/src/server/test/fixtures/audio.mp3 b/src/server/test/fixtures/audio.mp3 deleted file mode 100644 index 19531804..00000000 Binary files a/src/server/test/fixtures/audio.mp3 and /dev/null differ diff --git a/src/server/test/fixtures/original.jpg b/src/server/test/fixtures/original.jpg deleted file mode 100644 index 5a3b430b..00000000 Binary files a/src/server/test/fixtures/original.jpg and /dev/null differ diff --git a/src/server/test/fixtures/preview.jpg b/src/server/test/fixtures/preview.jpg deleted file mode 100644 index 19bfae44..00000000 Binary files a/src/server/test/fixtures/preview.jpg and /dev/null differ diff --git a/src/server/test/fixtures/thumbnail.jpg b/src/server/test/fixtures/thumbnail.jpg deleted file mode 100644 index facee563..00000000 Binary files a/src/server/test/fixtures/thumbnail.jpg and /dev/null differ diff --git a/src/server/test/list-projects-endpoint.js b/src/server/test/list-projects-endpoint.js deleted file mode 100644 index 3d91d49e..00000000 --- a/src/server/test/list-projects-endpoint.js +++ /dev/null @@ -1,73 +0,0 @@ -import { keyToPublicId as projectKeyToPublicId } from '@mapeo/crypto' -import assert from 'node:assert/strict' -import test from 'node:test' -import { - BEARER_TOKEN, - createTestServer, - randomAddProjectBody, -} from './test-helpers.js' - -test('listing projects', async (t) => { - const server = createTestServer(t, { allowedProjects: 999 }) - - await t.test('with invalid auth', async () => { - const response = await server.inject({ - method: 'GET', - url: '/projects', - headers: { Authorization: 'Bearer bad' }, - }) - assert.equal(response.statusCode, 403) - }) - - await t.test('with no projects', async () => { - const response = await server.inject({ - method: 'GET', - url: '/projects', - headers: { Authorization: 'Bearer ' + BEARER_TOKEN }, - }) - assert.equal(response.statusCode, 200) - assert.deepEqual(response.json(), { data: [] }) - }) - - await t.test('with projects', async () => { - const body1 = randomAddProjectBody() - const body2 = randomAddProjectBody() - - await Promise.all( - [body1, body2].map(async (body) => { - const response = await server.inject({ - method: 'PUT', - url: '/projects', - body, - }) - assert.equal(response.statusCode, 200) - }) - ) - - const response = await server.inject({ - method: 'GET', - url: '/projects', - headers: { Authorization: 'Bearer ' + BEARER_TOKEN }, - }) - assert.equal(response.statusCode, 200) - - const { data } = response.json() - assert(Array.isArray(data)) - assert.equal(data.length, 2, 'expected 2 projects') - for (const body of [body1, body2]) { - const projectPublicId = projectKeyToPublicId( - Buffer.from(body.projectKey, 'hex') - ) - /** @type {Record} */ - const project = data.find( - (project) => project.projectId === projectPublicId - ) - assert(project, `expected ${projectPublicId} to be found`) - assert.equal( - project.name, - body.projectName, - 'expected project name to match' - ) - } - }) -}) diff --git a/src/server/test/observations-endpoint.js b/src/server/test/observations-endpoint.js deleted file mode 100644 index 6e717c30..00000000 --- a/src/server/test/observations-endpoint.js +++ /dev/null @@ -1,280 +0,0 @@ -import { valueOf } from '@comapeo/schema' -import { keyToPublicId as projectKeyToPublicId } from '@mapeo/crypto' -import { generate } from '@mapeo/mock-data' -import { map } from 'iterpal' -import assert from 'node:assert/strict' -import * as fs from 'node:fs/promises' -import test from 'node:test' -import { setTimeout as delay } from 'node:timers/promises' -import { MapeoManager } from '../../index.js' -import { - BEARER_TOKEN, - createTestServer, - getManagerOptions, - randomAddProjectBody, -} from './test-helpers.js' -/** @import { ObservationValue } from '@comapeo/schema'*/ -/** @import { FastifyInstance } from 'fastify' */ - -const FIXTURES_ROOT = new URL('./fixtures/', import.meta.url) -const FIXTURE_ORIGINAL_PATH = new URL('original.jpg', FIXTURES_ROOT).pathname -const FIXTURE_PREVIEW_PATH = new URL('preview.jpg', FIXTURES_ROOT).pathname -const FIXTURE_THUMBNAIL_PATH = new URL('thumbnail.jpg', FIXTURES_ROOT).pathname -const FIXTURE_AUDIO_PATH = new URL('audio.mp3', FIXTURES_ROOT).pathname - -test('returns a 403 if no auth is provided', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'GET', - url: `/projects/${randomProjectPublicId()}/observations`, - }) - assert.equal(response.statusCode, 403) -}) - -test('returns a 403 if incorrect auth is provided', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'GET', - url: `/projects/${randomProjectPublicId()}/observations`, - headers: { Authorization: 'Bearer bad' }, - }) - assert.equal(response.statusCode, 403) -}) - -test('returning no observations', async (t) => { - const server = createTestServer(t) - const projectKeys = randomAddProjectBody() - const projectPublicId = projectKeyToPublicId( - Buffer.from(projectKeys.projectKey, 'hex') - ) - - const addProjectResponse = await server.inject({ - method: 'PUT', - url: '/projects', - body: projectKeys, - }) - assert.equal(addProjectResponse.statusCode, 200) - - const response = await server.inject({ - method: 'GET', - url: `/projects/${projectPublicId}/observations`, - headers: { Authorization: 'Bearer ' + BEARER_TOKEN }, - }) - assert.equal(response.statusCode, 200) - assert.deepEqual(await response.json(), { data: [] }) -}) - -test('returning observations with fetchable attachments', async (t) => { - const server = createTestServer(t) - - const serverAddress = await server.listen() - const serverUrl = new URL(serverAddress) - - const manager = new MapeoManager(getManagerOptions()) - const projectId = await manager.createProject({ name: 'CoMapeo project' }) - const project = await manager.getProject(projectId) - - await project.$member.addServerPeer(serverAddress, { - dangerouslyAllowInsecureConnections: true, - }) - - project.$sync.start() - project.$sync.connectServers() - - const observations = await Promise.all([ - (() => { - /** @type {ObservationValue} */ - const noAttachments = { - ...valueOf(generate('observation')[0]), - attachments: [], - } - return project.observation.create(noAttachments) - })(), - (async () => { - const { docId } = await project.observation.create( - valueOf(generate('observation')[0]) - ) - return project.observation.delete(docId) - })(), - (async () => { - const [imageBlob, audioBlob] = await Promise.all([ - project.$blobs.create( - { - original: FIXTURE_ORIGINAL_PATH, - preview: FIXTURE_PREVIEW_PATH, - thumbnail: FIXTURE_THUMBNAIL_PATH, - }, - { mimeType: 'image/jpeg', timestamp: Date.now() } - ), - project.$blobs.create( - { original: FIXTURE_AUDIO_PATH }, - { mimeType: 'audio/mpeg', timestamp: Date.now() } - ), - ]) - /** @type {ObservationValue} */ - const withAttachment = { - ...valueOf(generate('observation')[0]), - attachments: [blobToAttachment(imageBlob), blobToAttachment(audioBlob)], - } - return project.observation.create(withAttachment) - })(), - ]) - - await project.$sync.waitForSync('full') - - // It's possible that the client thinks it's synced but the server hasn't - // processed everything yet, so we try a few times. - const data = await runWithRetries(3, async () => { - const response = await server.inject({ - authority: serverUrl.host, - method: 'GET', - url: `/projects/${projectId}/observations`, - headers: { Authorization: 'Bearer ' + BEARER_TOKEN }, - }) - assert.equal(response.statusCode, 200) - - const { data } = await response.json() - assert.equal(data.length, 3) - return data - }) - - await Promise.all( - observations.map(async (observation) => { - const observationFromApi = data.find( - (/** @type {{ docId: string }} */ o) => o.docId === observation.docId - ) - assert(observationFromApi, 'observation found in API response') - assert.equal(observationFromApi.createdAt, observation.createdAt) - assert.equal(observationFromApi.updatedAt, observation.updatedAt) - assert.equal(observationFromApi.lat, observation.lat) - assert.equal(observationFromApi.lon, observation.lon) - assert.equal(observationFromApi.deleted, observation.deleted) - if (!observationFromApi.deleted) { - await assertAttachmentsCanBeFetchedAsJpeg({ - server, - serverAddress, - observationFromApi, - }) - } - assert.deepEqual(observationFromApi.tags, observation.tags) - }) - ) -}) - -function randomProjectPublicId() { - return projectKeyToPublicId( - Buffer.from(randomAddProjectBody().projectKey, 'hex') - ) -} - -/** - * @param {object} blob - * @param {string} blob.driveId - * @param {'photo' | 'audio' | 'video'} blob.type - * @param {string} blob.name - * @param {string} blob.hash - */ -function blobToAttachment(blob) { - return { - driveDiscoveryId: blob.driveId, - type: blob.type, - name: blob.name, - hash: blob.hash, - } -} - -/** - * @template T - * @param {number} retries - * @param {() => Promise} fn - * @returns {Promise} - */ -async function runWithRetries(retries, fn) { - for (let i = 0; i < retries - 1; i++) { - try { - return await fn() - } catch { - await delay(500) - } - } - return fn() -} - -/** - * @param {object} options - * @param {FastifyInstance} options.server - * @param {string} options.serverAddress - * @param {Record} options.observationFromApi - * @returns {Promise} - */ -async function assertAttachmentsCanBeFetchedAsJpeg({ - server, - serverAddress, - observationFromApi, -}) { - assert(Array.isArray(observationFromApi.attachments)) - await Promise.all( - observationFromApi.attachments.map( - /** @param {unknown} attachment */ - async (attachment) => { - assert(attachment && typeof attachment === 'object') - assert('url' in attachment && typeof attachment.url === 'string') - await assertAttachmentAndVariantsCanBeFetched( - server, - serverAddress, - attachment.url - ) - } - ) - ) -} - -/** - * @param {FastifyInstance} server - * @param {string} serverAddress - * @param {string} url - * @returns {Promise} - */ -async function assertAttachmentAndVariantsCanBeFetched( - server, - serverAddress, - url -) { - assert(url.startsWith(serverAddress)) - - /** @type {Map} */ - const variantsToCheck = new Map([ - [null, FIXTURE_ORIGINAL_PATH], - ['original', FIXTURE_ORIGINAL_PATH], - ['preview', FIXTURE_PREVIEW_PATH], - ['thumbnail', FIXTURE_THUMBNAIL_PATH], - ]) - - await Promise.all( - map(variantsToCheck, async ([variant, fixturePath]) => { - const expectedResponseBodyPromise = fs.readFile(fixturePath) - const attachmentResponse = await server.inject({ - method: 'GET', - url: url + (variant ? `?variant=${variant}` : ''), - headers: { Authorization: 'Bearer ' + BEARER_TOKEN }, - }) - assert.equal( - attachmentResponse.statusCode, - 200, - `expected 200 when fetching ${variant} attachment` - ) - assert.equal( - attachmentResponse.headers['content-type'], - 'image/jpeg', - `expected ${variant} attachment to be a JPEG` - ) - assert.deepEqual( - attachmentResponse.rawPayload, - await expectedResponseBodyPromise, - `expected ${variant} attachment to match fixture` - ) - }) - ) -} diff --git a/src/server/test/root.js b/src/server/test/root.js deleted file mode 100644 index 9d4422ee..00000000 --- a/src/server/test/root.js +++ /dev/null @@ -1,20 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { createTestServer } from './test-helpers.js' - -test('server root', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'GET', - url: '/', - }) - - assert.equal(response.statusCode, 200) - const contentType = response.headers['content-type'] - assert( - typeof contentType === 'string' && contentType.startsWith('text/html'), - 'response is HTML' - ) - assert(response.body.includes(' { - const serverName = 'test server' - const server = createTestServer(t, { serverName }) - - const response = await server.inject({ - method: 'GET', - url: '/info', - }) - - assert.equal(response.statusCode, 200) - assert.deepEqual(response.json(), { - data: { - deviceId: server.deviceId, - name: serverName, - }, - }) -}) diff --git a/src/server/test/sync-endpoint.js b/src/server/test/sync-endpoint.js deleted file mode 100644 index d2eaefaa..00000000 --- a/src/server/test/sync-endpoint.js +++ /dev/null @@ -1,53 +0,0 @@ -import { keyToPublicId as projectKeyToPublicId } from '@mapeo/crypto' -import assert from 'node:assert/strict' -import test from 'node:test' -import { createTestServer, randomAddProjectBody } from './test-helpers.js' - -test('sync endpoint is available after adding a project', async (t) => { - const server = createTestServer(t) - const addProjectBody = randomAddProjectBody() - const projectPublicId = projectKeyToPublicId( - Buffer.from(addProjectBody.projectKey, 'hex') - ) - - await t.test('sync endpoint not available yet', async () => { - const response = await server.inject({ - method: 'GET', - url: '/sync/' + projectPublicId, - headers: { - connection: 'upgrade', - upgrade: 'websocket', - }, - }) - assert.equal(response.statusCode, 404) - assert.equal(response.json().error, 'Not Found') - }) - - await server.inject({ - method: 'PUT', - url: '/projects', - body: addProjectBody, - }) - - await t.test('sync endpoint available', async (t) => { - const ws = await server.injectWS('/sync/' + projectPublicId) - t.after(() => ws.terminate()) - assert.equal(ws.readyState, ws.OPEN, 'websocket is open') - }) -}) - -test('sync endpoint returns error with an invalid project public ID', async (t) => { - const server = createTestServer(t) - - const response = await server.inject({ - method: 'GET', - url: '/sync/foobidoobi', - headers: { - connection: 'upgrade', - upgrade: 'websocket', - }, - }) - - assert.equal(response.statusCode, 400) - assert.equal(response.json().code, 'FST_ERR_VALIDATION') -}) diff --git a/src/server/test/test-helpers.js b/src/server/test/test-helpers.js deleted file mode 100644 index 26a9bd5e..00000000 --- a/src/server/test/test-helpers.js +++ /dev/null @@ -1,73 +0,0 @@ -import { KeyManager } from '@mapeo/crypto' -import createFastify from 'fastify' -import { randomBytes } from 'node:crypto' -import RAM from 'random-access-memory' -import comapeoServer from '../app.js' -/** @import { MapeoManager } from '../../index.js' */ -/** @import { TestContext } from 'node:test' */ -/** @import { ServerOptions } from '../app.js' */ - -export const BEARER_TOKEN = Buffer.from('swordfish').toString('base64') - -const TEST_SERVER_DEFAULTS = { - serverName: 'test server', - serverBearerToken: BEARER_TOKEN, -} - -/** - * @returns {ConstructorParameters[0]} - */ -export function getManagerOptions() { - const comapeoCoreUrl = new URL('../../..', import.meta.url) - const projectMigrationsFolder = new URL('./drizzle/project', comapeoCoreUrl) - .pathname - const clientMigrationsFolder = new URL('./drizzle/client', comapeoCoreUrl) - .pathname - return { - rootKey: randomBytes(16), - projectMigrationsFolder, - clientMigrationsFolder, - dbFolder: ':memory:', - coreStorage: () => new RAM(), - fastify: createFastify(), - } -} - -/** - * @param {TestContext} t - * @param {Partial} [serverOptions] - * @returns {import('fastify').FastifyInstance & { deviceId: string }} - */ -export function createTestServer(t, serverOptions) { - const managerOptions = getManagerOptions() - const km = new KeyManager(managerOptions.rootKey) - const server = createFastify() - server.register(comapeoServer, { - ...managerOptions, - ...TEST_SERVER_DEFAULTS, - ...serverOptions, - }) - t.after(() => server.close()) - Object.defineProperty(server, 'deviceId', { - get() { - return km.getIdentityKeypair().publicKey.toString('hex') - }, - }) - // @ts-expect-error - return server -} - -export const randomHex = (length = 32) => - Buffer.from(randomBytes(length)).toString('hex') - -export const randomAddProjectBody = () => ({ - projectName: randomHex(16), - projectKey: randomHex(), - encryptionKeys: { - auth: randomHex(), - config: randomHex(), - data: randomHex(), - blobIndex: randomHex(), - blob: randomHex(), - }, -}) diff --git a/src/server/types.ts b/src/server/types.ts deleted file mode 100644 index 923af9e9..00000000 --- a/src/server/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -// This file should be read by Typescript and augments the FastifyInstance -// Unfortunately it does this globally, which is a limitation of fastify -// typescript support currently, so need to be careful about using this where it -// is not in scope. - -import { type MapeoManager } from '../index.js' - -declare module 'fastify' { - interface FastifyInstance { - comapeo: MapeoManager - } - interface FastifyRequest { - baseUrl: URL - } -} diff --git a/src/server/ws-core-replicator.js b/src/server/ws-core-replicator.js deleted file mode 100644 index c2d2cf5a..00000000 --- a/src/server/ws-core-replicator.js +++ /dev/null @@ -1,58 +0,0 @@ -import { Transform } from 'node:stream' -import { pipeline } from 'node:stream/promises' -import { createWebSocketStream } from 'ws' -/** @import Protomux from 'protomux' */ -/** @import NoiseStream from '@hyperswarm/secret-stream' */ -/** @import { Duplex } from 'streamx' */ - -/** - * @internal - * @typedef {Omit & { userData: Protomux }} ProtocolStream - */ - -/** - * @internal - * @typedef {Duplex & { noiseStream: ProtocolStream }} ReplicationStream - */ - -/** - * @param {import('ws').WebSocket} ws - * @param {ReplicationStream} replicationStream - * @returns {Promise} - */ -export function wsCoreReplicator(ws, replicationStream) { - // This is purely to satisfy typescript at its worst. `pipeline` expects a - // NodeJS ReadWriteStream, but our replicationStream is a streamx Duplex - // stream. The difference is that streamx does not implement the - // `setEncoding`, `unpipe`, `wrap` or `isPaused` methods. The `pipeline` - // function does not depend on any of these methods (I have read through the - // NodeJS source code at cebf21d (v22.9.0) to confirm this), so we can safely - // cast the stream to a NodeJS ReadWriteStream. - const _replicationStream = /** @type {NodeJS.ReadWriteStream} */ ( - /** @type {unknown} */ (replicationStream) - ) - return pipeline( - _replicationStream, - wsSafetyTransform(ws), - createWebSocketStream(ws), - _replicationStream - ) -} - -/** - * Avoid writing data to a closing or closed websocket, which would result in an - * error. Instead we drop the data and wait for the stream close/end events to - * propagate and close the streams cleanly. - * - * @param {import('ws').WebSocket} ws - */ -function wsSafetyTransform(ws) { - return new Transform({ - transform(chunk, encoding, callback) { - if (ws.readyState === ws.CLOSED || ws.readyState === ws.CLOSING) { - return callback() - } - callback(null, chunk) - }, - }) -} diff --git a/test-e2e/server.js b/test-e2e/server.js index 7bd73126..bf740d5a 100644 --- a/test-e2e/server.js +++ b/test-e2e/server.js @@ -1,31 +1,9 @@ -import { valueOf } from '@comapeo/schema' -import { generate } from '@mapeo/mock-data' -import { execa } from 'execa' import createFastify from 'fastify' import assert from 'node:assert/strict' -import { randomBytes } from 'node:crypto' -import test, { mock } from 'node:test' -import { setTimeout as delay } from 'node:timers/promises' -import pDefer from 'p-defer' -import { pEvent } from 'p-event' -import RAM from 'random-access-memory' -import { MEMBER_ROLE_ID } from '../src/roles.js' -import comapeoServer from '../src/server/app.js' -import { - connectPeers, - createManager, - createManagers, - invite, - waitForPeers, - waitForSync, -} from './utils.js' -/** @import { FastifyInstance } from 'fastify' */ -/** @import { MapeoManager } from '../src/mapeo-manager.js' */ +import test from 'node:test' +import { createManager } from './utils.js' /** @import { MapeoProject } from '../src/mapeo-project.js' */ /** @import { MemberInfo } from '../src/member-api.js' */ -/** @import { State as SyncState } from '../src/sync/sync-api.js' */ - -const USE_REMOTE_SERVER = Boolean(process.env.REMOTE_TEST_SERVER) test('invalid base URLs', async (t) => { const manager = createManager('device0', t) @@ -216,255 +194,6 @@ test("fails if first request succeeds but sync doesn't", async (t) => { ) }) -test('adding a server peer', async (t) => { - const manager = createManager('device0', t) - const projectId = await manager.createProject({ name: 'foo' }) - const project = await manager.getProject(projectId) - - const { serverBaseUrl } = await createTestServer(t) - - assert(!(await findServerPeer(project)), 'no server peers before adding') - - await project.$member.addServerPeer(serverBaseUrl, { - dangerouslyAllowInsecureConnections: true, - }) - - const serverPeer = await findServerPeer(project) - assert(serverPeer, 'expected a server peer to be found by the client') - assert.equal(serverPeer.name, 'test server', 'server peers have name') - assert.equal( - serverPeer.role.roleId, - MEMBER_ROLE_ID, - 'server peers are added as regular members' - ) - assert.deepEqual( - new URL(serverPeer.selfHostedServerDetails?.baseUrl || ''), - new URL(serverBaseUrl), - 'server peer stores base URL' - ) -}) - -test("can't add a server to two different projects", async (t) => { - const [managerA, managerB] = await createManagers(2, t, 'mobile') - const projectIdA = await managerA.createProject({ name: 'project A' }) - const projectIdB = await managerB.createProject({ name: 'project B' }) - const projectA = await managerA.getProject(projectIdA) - const projectB = await managerB.getProject(projectIdB) - - const { serverBaseUrl } = await createTestServer(t) - - await projectA.$member.addServerPeer(serverBaseUrl, { - dangerouslyAllowInsecureConnections: true, - }) - - await assert.rejects(async () => { - await projectB.$member.addServerPeer(serverBaseUrl, { - dangerouslyAllowInsecureConnections: true, - }) - }, Error) -}) - -test('data can be synced via a server', async (t) => { - const [managers, { serverBaseUrl }] = await Promise.all([ - createManagers(2, t, 'mobile'), - createTestServer(t), - ]) - const [managerA, managerB] = managers - - // Manager A: create project and add the server to it - const projectId = await managerA.createProject({ name: 'foo' }) - const managerAProject = await managerA.getProject(projectId) - t.after(() => managerAProject.$sync.disconnectServers()) - await managerAProject.$member.addServerPeer(serverBaseUrl, { - dangerouslyAllowInsecureConnections: true, - }) - const serverDeviceIdPromise = findServerPeer(managerAProject).then( - (serverMember) => { - assert(serverMember, 'Manager A must have a server member') - return serverMember.deviceId - } - ) - - // Add Manager B to project - const disconnectManagers = connectPeers(managers) - t.after(disconnectManagers) - await waitForPeers(managers) - await invite({ invitor: managerA, invitees: [managerB], projectId }) - const managerBProject = await managerB.getProject(projectId) - t.after(() => managerBProject.$sync.disconnectServers()) - - // Sync managers to tell Manager B about the server - const projects = [managerAProject, managerBProject] - await waitForSync(projects, 'initial') - const serverPeer = await findServerPeer(managerBProject) - assert(serverPeer, 'expected a server peer to be found by the client') - - // Manager A adds data that Manager B doesn't know about - await disconnectManagers() - await waitForNoPeersToBeConnected(managerA) - await waitForNoPeersToBeConnected(managerB) - managerAProject.$sync.start() - managerAProject.$sync.connectServers() - const observation = await managerAProject.observation.create( - valueOf(generate('observation')[0]) - ) - const serverDeviceId = await serverDeviceIdPromise - await waitForSyncWithServer(managerAProject, serverDeviceId) - managerAProject.$sync.disconnectServers() - managerAProject.$sync.stop() - await assert.rejects( - () => managerBProject.observation.getByDocId(observation.docId), - "manager B doesn't see observation yet" - ) - - // Manager B sees observation after syncing - managerBProject.$sync.connectServers() - managerBProject.$sync.start() - await waitForSyncWithServer(managerBProject, serverDeviceId) - assert( - await managerBProject.observation.getByDocId(observation.docId), - 'manager B now sees data' - ) -}) - -test('connecting and then immediately disconnecting (and then immediately connecting again)', async (t) => { - const manager = createManager('seed', t) - await manager.setDeviceInfo({ name: 'manager', deviceType: 'mobile' }) - - // Because we need to stop the server, we can't use a remote server here. - const { server, serverBaseUrl } = await createLocalTestServer(t) - - const projectId = await manager.createProject({ name: 'foo' }) - const project = await manager.getProject(projectId) - await project.$member.addServerPeer(serverBaseUrl, { - dangerouslyAllowInsecureConnections: true, - }) - assert(await findServerPeer(project), 'test setup: server peer exists') - - await server.close() - - const bogusServer = createFastify() - const madeAnyRequestToServer = pDefer() - const anyRequestHandler = mock.fn(() => { - madeAnyRequestToServer.resolve() - return 'some request was made' - }) - bogusServer.all('*', anyRequestHandler) - const { port } = new URL(serverBaseUrl) - const bogusServerAddress = await bogusServer.listen({ port: Number(port) }) - t.after(() => bogusServer.close()) - assert.equal( - bogusServerAddress, - serverBaseUrl, - 'test setup: bogus server should have same address as "real" test server' - ) - - project.$sync.connectServers() - project.$sync.disconnectServers() - - await delay(500) - - assert.strictEqual( - anyRequestHandler.mock.calls.length, - 0, - 'no connection was made to the server' - ) - - project.$sync.connectServers() - project.$sync.disconnectServers() - project.$sync.connectServers() - - await madeAnyRequestToServer.promise - - assert.strictEqual( - anyRequestHandler.mock.calls.length, - 1, - 'a connection was made to the server' - ) -}) - -/** - * @typedef {object} LocalTestServer - * @prop {'local'} type - * @prop {string} serverBaseUrl - * @prop {FastifyInstance} server - */ - -/** - * @typedef {object} RemoteTestServer - * @prop {'remote'} type - * @prop {string} serverBaseUrl - */ - -/** - * @param {import('node:test').TestContext} t - * @returns {Promise} - */ -async function createTestServer(t) { - if (USE_REMOTE_SERVER) { - return createRemoteTestServer(t) - } else { - return createLocalTestServer(t) - } -} - -/** - * @param {import('node:test').TestContext} t - * @returns {Promise} - */ -async function createRemoteTestServer(t) { - const appName = 'comapeo-cloud-test-' + Math.random().toString(36).slice(8) - await execa( - 'flyctl', - ['apps', 'create', '--name', appName, '--org', 'digidem', '--json'], - { stdio: 'inherit' } - ) - t.after(async () => { - await execa('flyctl', ['apps', 'destroy', appName, '-y'], { - stdio: 'inherit', - }) - }) - await execa( - 'flyctl', - ['secrets', 'set', 'SERVER_BEARER_TOKEN=ignored', '--app', appName], - { stdio: 'inherit' } - ) - await execa( - 'flyctl', - ['deploy', '--app', appName, '-e', 'SERVER_NAME=test server'], - { - stdio: 'inherit', - } - ) - return { type: 'remote', serverBaseUrl: `https://${appName}.fly.dev/` } -} - -/** - * @param {import('node:test').TestContext} t - * @returns {Promise} - */ -async function createLocalTestServer(t) { - const comapeoCoreUrl = new URL('..', import.meta.url) - const projectMigrationsFolder = new URL('./drizzle/project', comapeoCoreUrl) - .pathname - const clientMigrationsFolder = new URL('./drizzle/client', comapeoCoreUrl) - .pathname - - const server = createFastify() - server.register(comapeoServer, { - rootKey: randomBytes(16), - projectMigrationsFolder, - clientMigrationsFolder, - dbFolder: ':memory:', - coreStorage: () => new RAM(), - serverName: 'test server', - serverBearerToken: 'ignored', - }) - const serverBaseUrl = await server.listen() - t.after(() => server.close()) - return { type: 'local', server, serverBaseUrl } -} - /** * @param {MapeoProject} project * @returns {Promise} @@ -474,52 +203,3 @@ async function findServerPeer(project) { (member) => member.deviceType === 'selfHostedServer' ) } - -/** - * @param {MapeoManager} manager - * @returns {Promise} - */ -function waitForNoPeersToBeConnected(manager) { - return new Promise((resolve) => { - const checkIfDone = async () => { - const isDone = (await manager.listLocalPeers()).every( - (peer) => peer.status === 'disconnected' - ) - if (isDone) { - manager.off('local-peers', checkIfDone) - resolve() - } - } - manager.on('local-peers', checkIfDone) - checkIfDone() - }) -} - -/** - * @param {MapeoProject} project - * @param {string} serverDeviceId - * @returns {Promise} - */ -async function waitForSyncWithServer(project, serverDeviceId) { - const initialState = project.$sync.getState() - if (isSyncedWithServer(initialState, serverDeviceId)) return - await pEvent(project.$sync, 'sync-state', (state) => - isSyncedWithServer(state, serverDeviceId) - ) -} - -/** - * @param {SyncState} syncState - * @param {string} serverDeviceId - * @returns {boolean} - */ -function isSyncedWithServer(syncState, serverDeviceId) { - const serverSyncState = syncState.remoteDeviceSyncState[serverDeviceId] - return Boolean( - serverSyncState && - serverSyncState.initial.want === 0 && - serverSyncState.initial.wanted === 0 && - serverSyncState.data.want === 0 && - serverSyncState.data.wanted === 0 - ) -}